From c39a373038c300a306dc2cf87bcfeebd898e4073 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 19 Nov 2024 17:21:25 +0200 Subject: [PATCH 01/21] added calculated field execution service --- .../cf/CalculatedFieldExecutionService.java | 25 ++ ...efaultCalculatedFieldExecutionService.java | 309 ++++++++++++++++++ .../entitiy/EntityStateSourcingListener.java | 16 +- .../entitiy/cf/CalculatedFieldCtx.java | 9 +- .../entitiy/cf/CalculatedFieldCtxId.java | 21 ++ .../entitiy/cf/CalculatedFieldResult.java | 46 +++ .../entitiy/cf/CalculatedFieldState.java | 9 +- .../cf/DefaultTbCalculatedFieldService.java | 258 --------------- .../service/entitiy/cf/RocksDBService.java | 2 + .../cf/ScriptCalculatedFieldState.java | 48 +++ .../cf/SimpleCalculatedFieldState.java | 37 ++- .../entitiy/cf/TbCalculatedFieldService.java | 4 - .../queue/DefaultTbCoreConsumerService.java | 10 +- .../CalculatedFieldControllerTest.java | 3 +- .../server/dao/cf/CalculatedFieldService.java | 2 + .../server/common/data/cf/Argument.java | 32 ++ .../cf/BaseCalculatedFieldConfiguration.java | 8 - .../data/cf/CalculatedFieldConfiguration.java | 2 +- .../dao/cf/BaseCalculatedFieldService.java | 5 + .../server/dao/cf/CalculatedFieldDao.java | 2 + .../dao/sql/cf/JpaCalculatedFieldDao.java | 5 + .../server/dao/service/AssetServiceTest.java | 3 +- .../service/CalculatedFieldServiceTest.java | 3 +- .../dao/service/CustomerServiceTest.java | 3 +- .../server/dao/service/DeviceServiceTest.java | 3 +- 25 files changed, 566 insertions(+), 299 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java new file mode 100644 index 0000000000..6daa88d771 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos; + +public interface CalculatedFieldExecutionService { + + void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java new file mode 100644 index 0000000000..c0eeec02bb --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -0,0 +1,309 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.entitiy.cf.CalculatedFieldCtx; +import org.thingsboard.server.service.entitiy.cf.CalculatedFieldCtxId; +import org.thingsboard.server.service.entitiy.cf.CalculatedFieldState; +import org.thingsboard.server.service.entitiy.cf.RocksDBService; +import org.thingsboard.server.service.entitiy.cf.ScriptCalculatedFieldState; +import org.thingsboard.server.service.entitiy.cf.SimpleCalculatedFieldState; +import org.thingsboard.server.service.partition.AbstractPartitionBasedService; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +@TbCoreComponent +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBasedService implements CalculatedFieldExecutionService { + + private final CalculatedFieldService calculatedFieldService; + private final AssetService assetService; + private final DeviceService deviceService; + private final AttributesService attributesService; + private final TimeseriesService timeseriesService; + private final RocksDBService rocksDBService; + + private ListeningExecutorService calculatedFieldExecutor; + private ListeningExecutorService calculatedFieldCallbackExecutor; + + private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap states = new ConcurrentHashMap<>(); + + @Value("${calculatedField.initFetchPackSize:50000}") + @Getter + private int initFetchPackSize; + + @PostConstruct + public void init() { + super.init(); + calculatedFieldExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); + calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); + scheduledExecutor.submit(this::fetchCalculatedFields); + } + + @PreDestroy + public void stop() { + if (calculatedFieldExecutor != null) { + calculatedFieldExecutor.shutdownNow(); + } + if (calculatedFieldCallbackExecutor != null) { + calculatedFieldCallbackExecutor.shutdownNow(); + } + } + + @Override + protected String getServiceName() { + return "Calculated Field Execution"; + } + + @Override + protected String getSchedulerExecutorName() { + return "calculated-field-scheduled"; + } + + @Override + protected Map>> onAddedPartitions(Set addedPartitions) { + // TODO: implementation for cluster mode + return Map.of(); + } + + @Override + protected void cleanupEntityOnPartitionRemoval(CalculatedFieldId entityId) { + // TODO: implementation for cluster mode + } + + @Override + public void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); + if (proto.getDeleted()) { + log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); + onCalculatedFieldDelete(calculatedFieldId, callback); + callback.onSuccess(); + } + CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); + if (proto.getUpdated()) { + log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); + boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); + if (!shouldReinit) { + return; + } + } + List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); + if (cf != null) { + EntityId entityId = cf.getEntityId(); + calculatedFields.put(calculatedFieldId, cf); + calculatedFieldLinks.put(calculatedFieldId, links); + switch (entityId.getEntityType()) { + case ASSET, DEVICE -> { + log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); + initializeStateForEntity(tenantId, cf, entityId, callback); + } + case ASSET_PROFILE -> { + log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); + PageDataIterable assetIds = new PageDataIterable<>(pageLink -> + assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); + assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, assetId, callback)); + } + case DEVICE_PROFILE -> { + log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); + PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> + deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); + deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, deviceId, callback)); + } + default -> + throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); + } + } else { + //Calculated field was probably deleted while message was in queue; + log.warn("Calculated field not found, possibly deleted: {}", calculatedFieldId); + callback.onSuccess(); + } + callback.onSuccess(); + log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); + } catch (Exception e) { + log.trace("Failed to process calculated field msg: [{}]", proto, e); + callback.onFailure(e); + } + } + + private boolean onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { + CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); + boolean shouldReinit = true; + if (hasSignificantChanges(oldCalculatedField, newCalculatedField)) { + onCalculatedFieldDelete(newCalculatedField.getId(), callback); + } else { + calculatedFields.put(newCalculatedField.getId(), newCalculatedField); + callback.onSuccess(); + shouldReinit = false; + } + return shouldReinit; + } + + private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { + try { + calculatedFieldLinks.remove(calculatedFieldId); + calculatedFields.remove(calculatedFieldId); + states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); + List statesToRemove = states.keySet().stream() + .filter(ctxId -> !calculatedFields.containsKey(new CalculatedFieldId(ctxId.cfId()))) + .map(JacksonUtil::writeValueAsString) + .toList(); + rocksDBService.deleteAll(statesToRemove); + } catch (Exception e) { + log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); + callback.onFailure(e); + } + } + + private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { + if (oldCalculatedField == null) { + return true; + } + boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); + boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); + CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); + CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); + boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); + boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); + boolean outputExpressionChanged = !oldConfig.getOutput().getExpression().equals(newConfig.getOutput().getExpression()); + + return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputExpressionChanged; + } + + private void fetchCalculatedFields() { + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); + PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); + cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldCtx.class))); + states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); + } + + private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { + Map arguments = calculatedField.getConfiguration().getArguments(); + Map argumentValues = new HashMap<>(); + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { + @Override + public void onSuccess(Optional result) { + String value = result.map(KvEntry::getValueAsString).orElse(argument.getDefaultValue()); + argumentValues.put(key, value); + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to initialize state for entity: [{}]", entityId, t); + callback.onFailure(t); + } + }, calculatedFieldCallbackExecutor)); + + updateOrInitializeState(calculatedField, entityId, argumentValues); + + } + + private ListenableFuture> fetchArgumentValue(TenantId tenantId, Argument argument) { + return switch (argument.getType()) { + case "ATTRIBUTES" -> Futures.transform( + attributesService.find(tenantId, argument.getEntityId(), AttributeScope.SERVER_SCOPE, argument.getKey()), + result -> result.map(entry -> (KvEntry) entry), + MoreExecutors.directExecutor()); + case "TIME_SERIES" -> Futures.transform( + timeseriesService.findLatest(tenantId, argument.getEntityId(), argument.getKey()), + result -> result.map(entry -> (KvEntry) entry), + MoreExecutors.directExecutor()); + default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); + }; + } + + private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { + CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(calculatedField.getUuidId(), entityId.getId()); + CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(ctxId, null)); + + CalculatedFieldState state = calculatedFieldCtx.getState(); + + if (state == null) { + state = createStateByType(calculatedField.getType()); + } + state.initState(argumentValues); + calculatedFieldCtx.setState(state); + states.put(ctxId, calculatedFieldCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); + + state.performCalculation(calculatedField.getConfiguration()); + } + + private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { + return switch (calculatedFieldType) { + case SIMPLE -> new SimpleCalculatedFieldState(); + case SCRIPT -> new ScriptCalculatedFieldState(); + }; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 6c2e995b82..1b541bcd5d 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -51,6 +51,8 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceCredentialsUpdateNotificationMsg; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; @@ -65,6 +67,8 @@ public class EntityStateSourcingListener { private final TbClusterService tbClusterService; private final TenantService tenantService; + private final CalculatedFieldService calculatedFieldService; + private final DeviceProfileService deviceProfileService; @PostConstruct public void init() { @@ -102,7 +106,7 @@ public class EntityStateSourcingListener { onTenantProfileUpdate(tenantProfile, lifecycleEvent); } case DEVICE -> { - onDeviceUpdate(event.getEntity(), event.getOldEntity()); + onDeviceUpdate(tenantId, event.getEntity(), event.getOldEntity()); } case DEVICE_PROFILE -> { DeviceProfile deviceProfile = (DeviceProfile) event.getEntity(); @@ -241,11 +245,19 @@ public class EntityStateSourcingListener { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); } - private void onDeviceUpdate(Object entity, Object oldEntity) { + private void onDeviceUpdate(TenantId tenantId, Object entity, Object oldEntity) { Device device = (Device) entity; Device oldDevice = null; if (oldEntity instanceof Device) { oldDevice = (Device) oldEntity; + // TODO: move verification of device type to cluster service + if (!oldDevice.getType().equals(device.getType())) { + DeviceProfile profile = deviceProfileService.findDeviceProfileByName(tenantId, device.getType()); + boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profile.getId()); + if (cfExistsByProfile) { + // TODO: send device type updated msg to core + } + } } tbClusterService.onDeviceUpdated(device, oldDevice); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java index 021a9bbc83..8a5e4cdf65 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java @@ -22,16 +22,15 @@ import org.thingsboard.server.common.data.id.EntityId; @Data public class CalculatedFieldCtx { - private CalculatedFieldId calculatedFieldId; - private EntityId entityId; + private CalculatedFieldCtxId id; private CalculatedFieldState state; public CalculatedFieldCtx() { } - public CalculatedFieldCtx(CalculatedFieldId calculatedFieldId, EntityId entityId, CalculatedFieldState state) { - this.calculatedFieldId = calculatedFieldId; - this.entityId = entityId; + public CalculatedFieldCtx(CalculatedFieldCtxId id, CalculatedFieldState state) { + this.id = id; this.state = state; } + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java new file mode 100644 index 0000000000..3dc0dead36 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import java.util.UUID; + +public record CalculatedFieldCtxId(UUID cfId, UUID entityId) { +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java new file mode 100644 index 0000000000..adb8b70e7e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; + +@Data +public class CalculatedFieldResult { + + private String name; + private String type; + private AttributeScope scope; + private String value; + + public static CalculatedFieldResult createAttributesResult(String name, AttributeScope scope, String value) { + CalculatedFieldResult result = new CalculatedFieldResult(); + result.name = name; + result.type = "ATTRIBUTES"; + result.scope = scope; + result.value = value; + return result; + } + + public static CalculatedFieldResult createTimeSeriesResult(String name, String value) { + CalculatedFieldResult result = new CalculatedFieldResult(); + result.name = name; + result.type = "TIME_SERIES"; + result.value = value; + return result; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java index 221c44b94c..9997df947f 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.entitiy.cf; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -36,6 +37,12 @@ public interface CalculatedFieldState { @JsonIgnore CalculatedFieldType getType(); - void performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration, boolean initialCalculation); + default boolean isValid(Map arguments, CalculatedFieldConfiguration calculatedFieldConfiguration) { + return arguments.keySet().containsAll(calculatedFieldConfiguration.getArguments().keySet()); + } + + void initState(Map argumentValues); + + CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index f2f16de844..0cc644606f 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -15,65 +15,28 @@ */ package org.thingsboard.server.service.entitiy.cf; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.ListeningScheduledExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; -import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.page.PageDataIterable; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; -import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateEntityId; @@ -84,109 +47,6 @@ import static org.thingsboard.server.dao.service.Validator.validateEntityId; public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { private final CalculatedFieldService calculatedFieldService; - private final AttributesService attributesService; - private final TimeseriesService timeseriesService; - private final RocksDBService rocksDBService; - private ListeningScheduledExecutorService scheduledExecutor; - - private ListeningExecutorService calculatedFieldExecutor; - private ListeningExecutorService calculatedFieldCallbackExecutor; - - private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); - private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); - private final ConcurrentMap states = new ConcurrentHashMap<>(); - - @Value("${calculatedField.initFetchPackSize:50000}") - @Getter - private int initFetchPackSize; - - @Value("10") - @Getter - private int defaultCalculatedFieldCheckIntervalInSec; - - @PostConstruct - public void init() { - // from AbstractPartitionBasedService - scheduledExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("calculated-field-scheduled"))); - /// - calculatedFieldExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( - Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); - calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( - Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.scheduleWithFixedDelay(this::fetchCalculatedFields, 0, defaultCalculatedFieldCheckIntervalInSec, TimeUnit.SECONDS); - } - - @PreDestroy - public void stop() { - // from AbstractPartitionBasedService - if (scheduledExecutor != null) { - scheduledExecutor.shutdown(); - } - /// - if (calculatedFieldExecutor != null) { - calculatedFieldExecutor.shutdownNow(); - } - if (calculatedFieldCallbackExecutor != null) { - calculatedFieldCallbackExecutor.shutdownNow(); - } - } - - @Override - public void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback) { - try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); - log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); - if (proto.getDeleted()) { - log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); - onCalculatedFieldDelete(calculatedFieldId, callback); - callback.onSuccess(); - } - CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); - if (proto.getUpdated()) { - log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); - boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); - if (!shouldReinit) { - return; - } - } - List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); - if (cf != null) { - EntityId entityId = cf.getEntityId(); - calculatedFields.put(calculatedFieldId, cf); - calculatedFieldLinks.put(calculatedFieldId, links); - switch (entityId.getEntityType()) { - case ASSET, DEVICE -> { - log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); - initializeStateForEntity(tenantId, cf, entityId, callback); - } - case ASSET_PROFILE -> { - log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); - PageDataIterable assetIds = new PageDataIterable<>(pageLink -> - assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, assetId, callback)); - } - case DEVICE_PROFILE -> { - log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); - PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> - deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, deviceId, callback)); - } - default -> - throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); - } - } else { - //Calculated field was probably deleted while message was in queue; - log.warn("Calculated field not found, possibly deleted: {}", calculatedFieldId); - callback.onSuccess(); - } - callback.onSuccess(); - log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); - } catch (Exception e) { - log.trace("Failed to process calculated field msg: [{}]", proto, e); - callback.onFailure(e); - } - } @Override public CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException { @@ -224,58 +84,6 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } - private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { - try { - calculatedFieldLinks.remove(calculatedFieldId); - calculatedFields.remove(calculatedFieldId); - states.keySet().removeIf(ctxId -> ctxId.startsWith(calculatedFieldId.getId().toString())); - List statesToRemove = states.keySet().stream() - .filter(key -> key.startsWith(calculatedFieldId.getId().toString())) - .collect(Collectors.toList()); - rocksDBService.deleteAll(statesToRemove); - } catch (Exception e) { - log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); - callback.onFailure(e); - } - } - - private boolean onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); - boolean shouldReinit = true; - if (hasSignificantChanges(oldCalculatedField, newCalculatedField)) { - onCalculatedFieldDelete(newCalculatedField.getId(), callback); - } else { - calculatedFields.put(newCalculatedField.getId(), newCalculatedField); - callback.onSuccess(); - shouldReinit = false; - } - return shouldReinit; - } - - private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { - if (oldCalculatedField == null) { - return true; - } - boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); - boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); - CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); - CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); - boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); - boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); - boolean outputExpressionChanged = !oldConfig.getOutput().getExpression().equals(newConfig.getOutput().getExpression()); - - return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputExpressionChanged; - } - - private void fetchCalculatedFields() { - PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); - cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); - PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); - cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(ctxId, JacksonUtil.fromString(ctx, CalculatedFieldCtx.class))); - states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.startsWith(id.toString()))); - } - private void checkEntityExistence(TenantId tenantId, EntityId entityId) { switch (entityId.getEntityType()) { case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> @@ -305,70 +113,4 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp }; } - private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { - Map arguments = calculatedField.getConfiguration().getArguments(); - Map argumentValues = new HashMap<>(); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { - @Override - public void onSuccess(Optional result) { - String value = result.map(KvEntry::getValueAsString).orElse(argument.getDefaultValue()); - argumentValues.put(key, value); - } - - @Override - public void onFailure(Throwable t) { - log.warn("Failed to initialize state for entity: [{}]", entityId, t); - callback.onFailure(t); - } - }, calculatedFieldCallbackExecutor)); - - updateOrInitializeState(calculatedField, entityId, argumentValues); - - } - - private ListenableFuture> fetchArgumentValue(TenantId tenantId, BaseCalculatedFieldConfiguration.Argument argument) { - return switch (argument.getType()) { - case "ATTRIBUTES" -> Futures.transform( - attributesService.find(tenantId, argument.getEntityId(), AttributeScope.SERVER_SCOPE, argument.getKey()), - result -> result.map(entry -> (KvEntry) entry), - MoreExecutors.directExecutor()); - case "TIME_SERIES" -> Futures.transform( - timeseriesService.findLatest(tenantId, argument.getEntityId(), argument.getKey()), - result -> result.map(entry -> (KvEntry) entry), - MoreExecutors.directExecutor()); - default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); - }; - } - - private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { - String ctxId = generateCtxId(calculatedField.getId(), entityId); - CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, - ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); - - CalculatedFieldState state = calculatedFieldCtx.getState(); - if (state != null) { - state.performCalculation(argumentValues, calculatedField.getConfiguration(), false); - } else { - CalculatedFieldState newState = createStateByType(calculatedField.getType()); - newState.performCalculation(argumentValues, calculatedField.getConfiguration(), true); - state = newState; - } - calculatedFieldCtx.setState(state); - - states.put(ctxId, calculatedFieldCtx); - rocksDBService.put(ctxId, Objects.requireNonNull(JacksonUtil.writeValueAsString(calculatedFieldCtx))); - } - - private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { - return switch (calculatedFieldType) { - case SIMPLE -> new SimpleCalculatedFieldState(); - default -> - throw new IllegalArgumentException("Invalid calculated field type '" + calculatedFieldType + "'."); - }; - } - - private String generateCtxId(CalculatedFieldId calculatedFieldId, EntityId entityId) { - return calculatedFieldId.getId() + "_" + entityId.getId(); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java index 2ba3e9bb4f..2cf7aec18d 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java @@ -21,6 +21,7 @@ import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; import org.rocksdb.WriteBatch; import org.rocksdb.WriteOptions; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.server.utils.RocksDBConfig; @@ -31,6 +32,7 @@ import java.util.Map; @Service @Slf4j +@ConditionalOnExpression("'${service.type:null}'=='monolith'") public class RocksDBService { private final RocksDB db; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java new file mode 100644 index 0000000000..52435643cc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.Data; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; + +import java.util.HashMap; +import java.util.Map; + +@Data +public class ScriptCalculatedFieldState implements CalculatedFieldState { + + private TbelInvokeService tbelInvokeService; + + private Map arguments = new HashMap<>(); + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SCRIPT; + } + + @Override + public void initState(Map argumentValues) { + + } + + @Override + public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { + return null; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java index 8a90893262..c004f9dd65 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java @@ -16,6 +16,8 @@ package org.thingsboard.server.service.entitiy.cf; import lombok.Data; +import net.objecthunter.exp4j.Expression; +import net.objecthunter.exp4j.ExpressionBuilder; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -26,8 +28,8 @@ import java.util.Map; public class SimpleCalculatedFieldState implements CalculatedFieldState { // TODO: use value object(TsKv) instead of string - Map arguments = new HashMap<>(); - String result; + private Map arguments = new HashMap<>(); + private String outputResult; @Override public CalculatedFieldType getType() { @@ -35,15 +37,30 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { } @Override - public void performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration, boolean initialCalculation) { - if (initialCalculation) { - // todo: perform initial calculation - this.arguments = argumentValues; - } else { - // todo: perform calculation based on previous data - this.arguments.putAll(argumentValues); + public void initState(Map argumentValues) { + this.arguments = argumentValues; + } + + @Override + public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { + if (isValid(arguments, calculatedFieldConfiguration)) { + String expression = calculatedFieldConfiguration.getOutput().getExpression(); + ThreadLocal customExpression = new ThreadLocal<>(); + var expr = customExpression.get(); + if (expr == null) { + expr = new ExpressionBuilder(expression) + .implicitMultiplication(true) + .variables(arguments.keySet()) + .build(); + customExpression.set(expr); + } + Map variables = new HashMap<>(); + arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v))); + expr.setVariables(variables); + double result = expr.evaluate(); + this.outputResult = Double.toString(result); } - this.result = "result"; + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java index aa77d29702..89931b8541 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java @@ -18,14 +18,10 @@ package org.thingsboard.server.service.entitiy.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.security.model.SecurityUser; public interface TbCalculatedFieldService { - void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException; CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 979c419003..2d3323e097 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -86,7 +86,7 @@ import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; -import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.notification.NotificationSchedulerService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -150,7 +150,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig> mainConsumer; private QueueConsumerManager> usageStatsConsumer; @@ -179,7 +179,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = deviceActivityEventsExecutor.submit(() -> calculatedFieldService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); + ListenableFuture future = deviceActivityEventsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index ebda2cbf82..57d7afcaba 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -21,6 +21,7 @@ import org.junit.Test; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.cf.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -136,7 +137,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); + Argument argument = new Argument(); argument.setEntityId(referencedEntityId); argument.setType("TIME_SERIES"); argument.setKey("temperature"); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index 4bb67f9f0d..e90f52c5c0 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -60,4 +60,6 @@ public interface CalculatedFieldService extends EntityDaoService { boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); + boolean existsCalculatedFieldByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java new file mode 100644 index 0000000000..bcd22f9216 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +public class Argument { + + private EntityId entityId; + private String key; + private String type; + private String defaultValue; + + private int limit; + private long timeWindow; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java index 4575e414ac..be69b951d1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java @@ -105,14 +105,6 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel return configNode; } - @Data - public static class Argument { - private EntityId entityId; - private String key; - private String type; - private String defaultValue; - } - @Data public static class Output { private String name; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java index f733c35310..a3598cb59d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -39,7 +39,7 @@ public interface CalculatedFieldConfiguration { @JsonIgnore CalculatedFieldType getType(); - Map getArguments(); + Map getArguments(); BaseCalculatedFieldConfiguration.Output getOutput(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index fd669f7865..1ee881ae72 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -195,6 +195,11 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements .anyMatch(referencedEntities -> referencedEntities.contains(referencedEntityId)); } + @Override + public boolean existsCalculatedFieldByEntityId(TenantId tenantId, EntityId entityId) { + return calculatedFieldDao.existsByEntityId(tenantId, entityId); + }; + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findById(tenantId, new CalculatedFieldId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index d1c9fd86bc..a6b7c2dea1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -34,4 +34,6 @@ public interface CalculatedFieldDao extends Dao { List removeAllByEntityId(TenantId tenantId, EntityId entityId); + boolean existsByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index bdc701070d..737a089a15 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -66,6 +66,11 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao getEntityClass() { return CalculatedFieldEntity.class; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 43b51ce2ac..2082210899 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; @@ -880,7 +881,7 @@ public class AssetServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); + Argument argument = new Argument(); argument.setEntityId(savedAsset.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index d5d025a46d..751d883896 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -23,6 +23,7 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.cf.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -149,7 +150,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); + Argument argument = new Argument(); argument.setEntityId(referencedEntityId); argument.setType("TIME_SERIES"); argument.setKey("temperature"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 1c5e0d8f49..73d398bb39 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -31,6 +31,7 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.cf.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; @@ -375,7 +376,7 @@ public class CustomerServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); + Argument argument = new Argument(); argument.setEntityId(savedCustomer.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 8f1f16e631..1bd876eae0 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.cf.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; @@ -1218,7 +1219,7 @@ public class DeviceServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); + Argument argument = new Argument(); argument.setEntityId(device.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); From 40299cab1ac2a54a73c09f342b48c5923d52d99b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 20 Nov 2024 17:34:07 +0200 Subject: [PATCH 02/21] added nofitication on update in telemetry service --- .../cf/CalculatedFieldExecutionService.java | 8 +++ .../cf/CalculatedFieldResult.java | 23 ++---- ...efaultCalculatedFieldExecutionService.java | 70 +++++++++++++++---- .../{entitiy => }/cf/RocksDBService.java | 4 +- .../cf => cf/ctx}/CalculatedFieldCtx.java | 5 +- .../cf => cf/ctx}/CalculatedFieldCtxId.java | 2 +- .../ctx/state}/CalculatedFieldState.java | 14 ++-- .../state}/ScriptCalculatedFieldState.java | 19 +++-- .../state}/SimpleCalculatedFieldState.java | 38 +++++++--- .../cf/DefaultTbCalculatedFieldService.java | 2 +- .../DefaultTelemetrySubscriptionService.java | 60 +++++++++++++++- .../CalculatedFieldControllerTest.java | 9 +-- .../server/dao/cf/CalculatedFieldService.java | 4 ++ .../common/data/cf/CalculatedField.java | 2 + .../cf/CalculatedFieldLinkConfiguration.java | 8 +-- .../data/cf/{ => configuration}/Argument.java | 4 +- .../BaseCalculatedFieldConfiguration.java | 43 ++++++------ .../CalculatedFieldConfiguration.java | 9 ++- .../common/data/cf/configuration/Output.java | 29 ++++++++ .../ScriptCalculatedFieldConfiguration.java | 40 +++++++++++ .../SimpleCalculatedFieldConfiguration.java | 3 +- .../dao/cf/BaseCalculatedFieldService.java | 16 ++++- .../server/dao/cf/CalculatedFieldLinkDao.java | 3 + .../dao/model/sql/CalculatedFieldEntity.java | 7 +- .../sql/cf/CalculatedFieldLinkRepository.java | 2 + ...efaultNativeCalculatedFieldRepository.java | 4 +- .../dao/sql/cf/JpaCalculatedFieldLinkDao.java | 6 ++ .../server/dao/service/AssetServiceTest.java | 7 +- .../service/CalculatedFieldServiceTest.java | 9 +-- .../dao/service/CustomerServiceTest.java | 7 +- .../server/dao/service/DeviceServiceTest.java | 7 +- 31 files changed, 349 insertions(+), 115 deletions(-) rename application/src/main/java/org/thingsboard/server/service/{entitiy => }/cf/CalculatedFieldResult.java (53%) rename application/src/main/java/org/thingsboard/server/service/{entitiy => }/cf/RocksDBService.java (98%) rename application/src/main/java/org/thingsboard/server/service/{entitiy/cf => cf/ctx}/CalculatedFieldCtx.java (84%) rename application/src/main/java/org/thingsboard/server/service/{entitiy/cf => cf/ctx}/CalculatedFieldCtxId.java (93%) rename application/src/main/java/org/thingsboard/server/service/{entitiy/cf => cf/ctx/state}/CalculatedFieldState.java (70%) rename application/src/main/java/org/thingsboard/server/service/{entitiy/cf => cf/ctx/state}/ScriptCalculatedFieldState.java (73%) rename application/src/main/java/org/thingsboard/server/service/{entitiy/cf => cf/ctx/state}/SimpleCalculatedFieldState.java (57%) rename common/data/src/main/java/org/thingsboard/server/common/data/cf/{ => configuration}/Argument.java (85%) rename common/data/src/main/java/org/thingsboard/server/common/data/cf/{ => configuration}/BaseCalculatedFieldConfiguration.java (81%) rename common/data/src/main/java/org/thingsboard/server/common/data/cf/{ => configuration}/CalculatedFieldConfiguration.java (81%) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java rename common/data/src/main/java/org/thingsboard/server/common/data/cf/{ => configuration}/SimpleCalculatedFieldConfiguration.java (90%) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 6daa88d771..9a8b26d074 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,11 +15,19 @@ */ package org.thingsboard.server.service.cf; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos; +import java.util.Map; + public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); + void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); + +// void onEntityProfileUpdate(TransportProtos.CalculatedFieldEntityProfileUpdateMsgProto proto, TbCallback callback); + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java similarity index 53% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java rename to application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index adb8b70e7e..4982445735 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -13,34 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf; import lombok.Data; import org.thingsboard.server.common.data.AttributeScope; +import java.util.Map; + @Data public class CalculatedFieldResult { - private String name; private String type; private AttributeScope scope; - private String value; - - public static CalculatedFieldResult createAttributesResult(String name, AttributeScope scope, String value) { - CalculatedFieldResult result = new CalculatedFieldResult(); - result.name = name; - result.type = "ATTRIBUTES"; - result.scope = scope; - result.value = value; - return result; - } + private Map resultMap; - public static CalculatedFieldResult createTimeSeriesResult(String name, String value) { - CalculatedFieldResult result = new CalculatedFieldResult(); - result.name = name; - result.type = "TIME_SERIES"; - result.value = value; - return result; + public CalculatedFieldResult() { } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index c0eeec02bb..250496830c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -29,13 +30,12 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.cf.Argument; -import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; +import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -44,7 +44,10 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.asset.AssetService; @@ -54,12 +57,11 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.entitiy.cf.CalculatedFieldCtx; -import org.thingsboard.server.service.entitiy.cf.CalculatedFieldCtxId; -import org.thingsboard.server.service.entitiy.cf.CalculatedFieldState; -import org.thingsboard.server.service.entitiy.cf.RocksDBService; -import org.thingsboard.server.service.entitiy.cf.ScriptCalculatedFieldState; -import org.thingsboard.server.service.entitiy.cf.SimpleCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import java.util.ArrayList; @@ -71,6 +73,9 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.thingsboard.server.common.data.DataConstants.SCOPE; @TbCoreComponent @Service @@ -84,6 +89,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final AttributesService attributesService; private final TimeseriesService timeseriesService; private final RocksDBService rocksDBService; + private final TbClusterService clusterService; private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; @@ -194,6 +200,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + @Override + public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + try { + CalculatedField calculatedField = calculatedFields.computeIfAbsent(calculatedFieldId, id -> calculatedFieldService.findById(tenantId, id)); + updateOrInitializeState(calculatedField, calculatedField.getEntityId(), updatedTelemetry); + log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); + } catch (Exception e) { + log.trace("Failed to update time series for calculatedFieldId: [{}]", calculatedFieldId, e); + } + } + private boolean onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); boolean shouldReinit = true; @@ -250,11 +267,15 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { Map arguments = calculatedField.getConfiguration().getArguments(); Map argumentValues = new HashMap<>(); + AtomicInteger remaining = new AtomicInteger(arguments.size()); arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { @Override public void onSuccess(Optional result) { String value = result.map(KvEntry::getValueAsString).orElse(argument.getDefaultValue()); argumentValues.put(key, value); + if (remaining.decrementAndGet() == 0) { + updateOrInitializeState(calculatedField, entityId, argumentValues); + } } @Override @@ -263,15 +284,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas callback.onFailure(t); } }, calculatedFieldCallbackExecutor)); - - updateOrInitializeState(calculatedField, entityId, argumentValues); - } private ListenableFuture> fetchArgumentValue(TenantId tenantId, Argument argument) { return switch (argument.getType()) { case "ATTRIBUTES" -> Futures.transform( - attributesService.find(tenantId, argument.getEntityId(), AttributeScope.SERVER_SCOPE, argument.getKey()), + attributesService.find(tenantId, argument.getEntityId(), argument.getScope(), argument.getKey()), result -> result.map(entry -> (KvEntry) entry), MoreExecutors.directExecutor()); case "TIME_SERIES" -> Futures.transform( @@ -296,7 +314,29 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.put(ctxId, calculatedFieldCtx); rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); - state.performCalculation(calculatedField.getConfiguration()); + CalculatedFieldResult result = state.performCalculation(calculatedField.getConfiguration()); + if (result != null) { + pushMsgToRuleEngine(calculatedField.getTenantId(), calculatedField.getEntityId(), result); + } + } + + private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { + try { + String type = calculatedFieldResult.getType(); + TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; + TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; + ObjectNode jsonNodes = createJsonPayload(calculatedFieldResult); + TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(jsonNodes)); + clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); + } catch (Exception e) { + log.warn("[{}] Failed to push message to rule engine. CalculatedFieldResult: {}", originatorId, calculatedFieldResult, e); + } + } + + private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { + ObjectNode jsonNodes = JacksonUtil.newObjectNode(); + calculatedFieldResult.getResultMap().forEach(jsonNodes::put); + return jsonNodes; } private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java similarity index 98% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java rename to application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java index 2cf7aec18d..d6b2980042 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf; import lombok.extern.slf4j.Slf4j; import org.rocksdb.RocksDB; @@ -94,4 +94,4 @@ public class RocksDBService { return map; } -} \ No newline at end of file +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java similarity index 84% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java index 8a5e4cdf65..4b2a6c918f 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java @@ -13,11 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf.ctx; import lombok.Data; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @Data public class CalculatedFieldCtx { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java index 3dc0dead36..a316c54b76 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf.ctx; import java.util.UUID; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java similarity index 70% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 9997df947f..c25a6960ac 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -13,14 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf.ctx.state; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.Map; @@ -30,15 +31,16 @@ import java.util.Map; property = "type" ) @JsonSubTypes({ - @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE") + @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE"), + @JsonSubTypes.Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT") }) public interface CalculatedFieldState { @JsonIgnore CalculatedFieldType getType(); - default boolean isValid(Map arguments, CalculatedFieldConfiguration calculatedFieldConfiguration) { - return arguments.keySet().containsAll(calculatedFieldConfiguration.getArguments().keySet()); + default boolean isValid(Map argumentValues, Map arguments) { + return argumentValues.keySet().containsAll(arguments.keySet()); } void initState(Map argumentValues); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java similarity index 73% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 52435643cc..238e8005f2 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -13,23 +13,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf.ctx.state; import lombok.Data; -import org.thingsboard.script.api.tbel.TbelInvokeService; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; import java.util.Map; @Data +@Slf4j public class ScriptCalculatedFieldState implements CalculatedFieldState { - private TbelInvokeService tbelInvokeService; - private Map arguments = new HashMap<>(); + public ScriptCalculatedFieldState() { + } + @Override public CalculatedFieldType getType() { return CalculatedFieldType.SCRIPT; @@ -37,11 +40,15 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { @Override public void initState(Map argumentValues) { - + if (arguments == null) { + this.arguments = new HashMap<>(); + } + this.arguments.putAll(argumentValues); } @Override public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { + // TODO: implement return null; } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java similarity index 57% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index c004f9dd65..e984e300c6 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -13,13 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf.ctx.state; import lombok.Data; import net.objecthunter.exp4j.Expression; import net.objecthunter.exp4j.ExpressionBuilder; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; import java.util.Map; @@ -28,8 +31,7 @@ import java.util.Map; public class SimpleCalculatedFieldState implements CalculatedFieldState { // TODO: use value object(TsKv) instead of string - private Map arguments = new HashMap<>(); - private String outputResult; + private Map arguments; @Override public CalculatedFieldType getType() { @@ -38,29 +40,43 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { @Override public void initState(Map argumentValues) { - this.arguments = argumentValues; + if (arguments == null) { + arguments = new HashMap<>(); + } + arguments.putAll(argumentValues); } @Override public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { - if (isValid(arguments, calculatedFieldConfiguration)) { - String expression = calculatedFieldConfiguration.getOutput().getExpression(); + Output output = calculatedFieldConfiguration.getOutput(); + Map arguments = calculatedFieldConfiguration.getArguments(); + + if (isValid(this.arguments, arguments)) { + CalculatedFieldResult result = new CalculatedFieldResult(); + String expression = output.getExpression(); ThreadLocal customExpression = new ThreadLocal<>(); var expr = customExpression.get(); if (expr == null) { expr = new ExpressionBuilder(expression) .implicitMultiplication(true) - .variables(arguments.keySet()) + .variables(this.arguments.keySet()) .build(); customExpression.set(expr); } Map variables = new HashMap<>(); - arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v))); + this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v))); expr.setVariables(variables); - double result = expr.evaluate(); - this.outputResult = Double.toString(result); + + String expressionResult = String.valueOf(expr.evaluate()); + + result.setType(output.getType()); + result.setScope(output.getScope()); + result.setResultMap(Map.of(output.getName(), expressionResult)); + return result; } + return null; + // TODO: handle what happens when not valid } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 0cc644606f..4d28ff55ac 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 5513c41929..ac769bee9e 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -32,6 +32,8 @@ import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -40,6 +42,7 @@ import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -47,9 +50,11 @@ import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.stats.TbApiUsageReportClient; import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -77,6 +82,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final TbEntityViewService tbEntityViewService; private final TbApiUsageReportClient apiUsageClient; private final TbApiUsageStateService apiUsageStateService; + private final CalculatedFieldService calculatedFieldService; + private final CalculatedFieldExecutionService calculatedFieldExecutionService; private ExecutorService tsCallBackExecutor; @@ -87,12 +94,16 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer TimeseriesService tsService, @Lazy TbEntityViewService tbEntityViewService, TbApiUsageReportClient apiUsageClient, - TbApiUsageStateService apiUsageStateService) { + TbApiUsageStateService apiUsageStateService, + CalculatedFieldService calculatedFieldService, + CalculatedFieldExecutionService calculatedFieldExecutionService) { this.attrService = attrService; this.tsService = tsService; this.tbEntityViewService = tbEntityViewService; this.apiUsageClient = apiUsageClient; this.apiUsageStateService = apiUsageStateService; + this.calculatedFieldService = calculatedFieldService; + this.calculatedFieldExecutionService = calculatedFieldExecutionService; } @PostConstruct @@ -179,6 +190,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer addMainCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); addEntityViewCallback(tenantId, entityId, ts); + updateTelemetryInCalculatedFields(tenantId, entityId, ts); } private void saveWithoutLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { @@ -187,6 +199,49 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); } + private void updateTelemetryInCalculatedFields(TenantId tenantId, EntityId entityId, List telemetry) { + if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { + List cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); + if (!cfLinks.isEmpty()) { + cfLinks.forEach(link -> { + CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); + Map attributes = link.getConfiguration().getAttributes(); + Map timeSeries = link.getConfiguration().getTimeSeries(); + List filteredTelemetry = telemetry.stream() + .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) + .toList(); + + + Map updatedTelemetry = new HashMap<>(); + for (KvEntry telemetryEntry : filteredTelemetry) { + String key = telemetryEntry.getKey(); + if (telemetryEntry instanceof AttributeKvEntry) { + for (Map.Entry attribute : attributes.entrySet()) { + if (telemetryEntry.getKey().equals(attribute.getValue())) { + key = attribute.getKey(); + break; + } + } + } + if (telemetryEntry instanceof TsKvEntry) { + for (Map.Entry timeSeriesEntry : timeSeries.entrySet()) { + if (telemetryEntry.getKey().equals(timeSeriesEntry.getValue())) { + key = timeSeriesEntry.getKey(); + break; + } + } + } + updatedTelemetry.put(key, telemetryEntry.getValueAsString()); + } + + if (!updatedTelemetry.isEmpty()) { + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, calculatedFieldId, updatedTelemetry); + } + }); + } + } + } + private void addEntityViewCallback(TenantId tenantId, EntityId entityId, List ts) { if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { Futures.addCallback(this.tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId), @@ -263,6 +318,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice)); + updateTelemetryInCalculatedFields(tenantId, entityId, attributes); } @Override @@ -270,6 +326,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope.name(), attributes, notifyDevice)); + updateTelemetryInCalculatedFields(tenantId, entityId, attributes); } @Override @@ -283,6 +340,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = tsService.saveLatest(tenantId, entityId, ts); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); + updateTelemetryInCalculatedFields(tenantId, entityId, ts); } @Override diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 57d7afcaba..314dc2bdba 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -21,11 +21,12 @@ import org.junit.Test; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.security.Authority; @@ -144,7 +145,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { config.setArguments(Map.of("T", argument)); - SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); + Output output = new Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index e90f52c5c0..e44ff0ba22 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -54,12 +54,16 @@ public interface CalculatedFieldService extends EntityDaoService { List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findAllCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + ListenableFuture> findAllCalculatedFieldLinksByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); PageData findAllCalculatedFieldLinks(PageLink pageLink); boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); + boolean referencedInAnyCalculatedFieldIncludingEntityId(TenantId tenantId, EntityId referencedEntityId); + boolean existsCalculatedFieldByEntityId(TenantId tenantId, EntityId entityId); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index ceb1222fe2..e626c9d3d2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -25,6 +25,8 @@ import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.HasVersion; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java index 02d668a67a..c5f81cd572 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java @@ -17,13 +17,13 @@ package org.thingsboard.server.common.data.cf; import lombok.Data; -import java.util.ArrayList; -import java.util.List; +import java.util.HashMap; +import java.util.Map; @Data public class CalculatedFieldLinkConfiguration { - private List attributes = new ArrayList<>(); - private List timeSeries = new ArrayList<>(); + private Map attributes = new HashMap<>(); + private Map timeSeries = new HashMap<>(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java similarity index 85% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index bcd22f9216..f34f5e9cb7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.cf; +package org.thingsboard.server.common.data.cf.configuration; import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.EntityId; @Data @@ -24,6 +25,7 @@ public class Argument { private EntityId entityId; private String key; private String type; + private AttributeScope scope; private String defaultValue; private int limit; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java similarity index 81% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index be69b951d1..7692d792f8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -13,14 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.cf; +package org.thingsboard.server.common.data.cf.configuration; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -60,18 +62,20 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel @Override public CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId) { CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); - arguments.values().stream() - .filter(argument -> argument.getEntityId().equals(entityId)) - .forEach(argument -> { - switch (argument.getType()) { - case "ATTRIBUTES": - linkConfiguration.getAttributes().add(argument.getKey()); - break; - case "TIME_SERIES": - linkConfiguration.getTimeSeries().add(argument.getKey()); - break; - } - }); + + for (Map.Entry entry : arguments.entrySet()) { + Argument argument = entry.getValue(); + if (argument.getEntityId().equals(entityId)) { + switch (argument.getType()) { + case "ATTRIBUTES": + linkConfiguration.getAttributes().put(entry.getKey(), argument.getKey()); + break; + case "TIME_SERIES": + linkConfiguration.getTimeSeries().put(entry.getKey(), argument.getKey()); + break; + } + } + } return linkConfiguration; } @@ -93,25 +97,21 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel } argumentNode.put("key", argument.getKey()); argumentNode.put("type", argument.getType()); + argumentNode.put("scope", String.valueOf(argument.getScope())); argumentNode.put("defaultValue", argument.getDefaultValue()); }); if (output != null) { ObjectNode outputNode = configNode.putObject("output"); + outputNode.put("name", output.getName()); outputNode.put("type", output.getType()); + outputNode.put("scope", String.valueOf(output.getScope())); outputNode.put("expression", output.getExpression()); } return configNode; } - @Data - public static class Output { - private String name; - private String type; - private String expression; - } - private BaseCalculatedFieldConfiguration toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { if (config == null || !config.isObject()) { return null; @@ -133,6 +133,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel } argument.setKey(argumentNode.get("key").asText()); argument.setType(argumentNode.get("type").asText()); + argument.setScope(AttributeScope.valueOf(argumentNode.get("scope").asText())); argument.setDefaultValue(argumentNode.get("defaultValue").asText()); arguments.put(key, argument); }); @@ -142,7 +143,9 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel JsonNode outputNode = config.get("output"); if (outputNode != null) { Output output = new Output(); + output.setName(outputNode.get("name").asText()); output.setType(outputNode.get("type").asText()); + output.setScope(AttributeScope.valueOf(outputNode.get("scope").asText())); output.setExpression(outputNode.get("expression").asText()); this.setOutput(output); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java similarity index 81% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index a3598cb59d..155015028f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -13,13 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.cf; +package org.thingsboard.server.common.data.cf.configuration; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.JsonNode; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.EntityId; import java.util.List; @@ -32,7 +34,8 @@ import java.util.UUID; property = "type" ) @JsonSubTypes({ - @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE") + @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE"), + @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT") }) public interface CalculatedFieldConfiguration { @@ -41,7 +44,7 @@ public interface CalculatedFieldConfiguration { Map getArguments(); - BaseCalculatedFieldConfiguration.Output getOutput(); + Output getOutput(); @JsonIgnore List getReferencedEntities(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java new file mode 100644 index 0000000000..683e372ebc --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; + +@Data +public class Output { + + private String name; + private String type; + private AttributeScope scope; + private String expression; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..a24328b4c9 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; + +import java.util.UUID; + +@Data +public class ScriptCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + public ScriptCalculatedFieldConfiguration() { + super(); + } + + public ScriptCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + super(config, entityType, entityId); + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SCRIPT; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java similarity index 90% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java index 327f9cdc75..af11d2f5d8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java @@ -13,11 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.cf; +package org.thingsboard.server.common.data.cf.configuration; import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import java.util.UUID; diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 1ee881ae72..cf5f79c5fd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -21,7 +21,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; @@ -173,6 +173,12 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldLinkDao.findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId); } + @Override + public List findAllCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing findAllCalculatedFieldLinksByEntityId, entityId [{}]", entityId); + return calculatedFieldLinkDao.findCalculatedFieldLinksByEntityId(tenantId, entityId); + } + @Override public ListenableFuture> findAllCalculatedFieldLinksByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { log.trace("Executing findAllCalculatedFieldLinksByIdAsync, calculatedFieldId [{}]", calculatedFieldId); @@ -195,6 +201,14 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements .anyMatch(referencedEntities -> referencedEntities.contains(referencedEntityId)); } + @Override + public boolean referencedInAnyCalculatedFieldIncludingEntityId(TenantId tenantId, EntityId referencedEntityId) { + return calculatedFieldDao.findAllByTenantId(tenantId).stream() + .map(CalculatedField::getConfiguration) + .map(CalculatedFieldConfiguration::getReferencedEntities) + .anyMatch(referencedEntities -> referencedEntities.contains(referencedEntityId)); + } + @Override public boolean existsCalculatedFieldByEntityId(TenantId tenantId, EntityId entityId) { return calculatedFieldDao.existsByEntityId(tenantId, entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java index 34f2129bd7..549db510ab 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.cf; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -29,6 +30,8 @@ public interface CalculatedFieldLinkDao extends Dao { List findCalculatedFieldLinksByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + ListenableFuture> findCalculatedFieldLinksByCalculatedFieldIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); List findAll(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 3c45a81cb5..6500d2a1e7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -24,9 +24,10 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -122,6 +123,8 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem switch (CalculatedFieldType.valueOf(type)) { case SIMPLE: return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + case SCRIPT: + return new ScriptCalculatedFieldConfiguration(config, entityType, entityId); default: throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java index 61c4026cca..d7325df8d1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java @@ -25,4 +25,6 @@ public interface CalculatedFieldLinkRepository extends JpaRepository findAllByTenantIdAndCalculatedFieldId(UUID tenantId, UUID calculatedFieldId); + List findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index 417a468b2c..eebca14b6e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -25,11 +25,11 @@ import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java index 417b529dc9..29492a10cb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -23,6 +23,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -49,6 +50,11 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao findCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { + return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId())); + } + @Override public ListenableFuture> findCalculatedFieldLinksByCalculatedFieldIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { return service.submit(() -> findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 2082210899..2012090264 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -30,10 +30,11 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; -import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -888,7 +889,7 @@ public class AssetServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); - SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); + Output output = new Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 751d883896..2bdb1b8897 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -23,11 +23,12 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.dao.cf.CalculatedFieldService; @@ -157,7 +158,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); - SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); + Output output = new Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 73d398bb39..b58f739462 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -31,10 +31,11 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -383,7 +384,7 @@ public class CustomerServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); - SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); + Output output = new Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 1bd876eae0..38bd21170a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -39,10 +39,11 @@ import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -1226,7 +1227,7 @@ public class DeviceServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); - SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); + Output output = new Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); From c75603f57c0d093d14cb7c3cf4fe187082f70ca5 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 21 Nov 2024 15:45:39 +0200 Subject: [PATCH 03/21] added implementation for script type --- .../cf/CalculatedFieldExecutionService.java | 5 +- .../service/cf/CalculatedFieldResult.java | 2 +- ...efaultCalculatedFieldExecutionService.java | 43 ++++++--- .../state/CalculatedFieldScriptEngine.java | 29 ++++++ .../cf/ctx/state/CalculatedFieldState.java | 12 ++- .../CalculatedFieldTbelScriptEngine.java | 90 +++++++++++++++++++ .../ctx/state/ScriptCalculatedFieldState.java | 59 ++++++++++-- .../ctx/state/SimpleCalculatedFieldState.java | 22 ++--- .../DefaultTelemetrySubscriptionService.java | 47 +++++----- .../thingsboard/script/api/ScriptType.java | 2 +- 10 files changed, 248 insertions(+), 63 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 9a8b26d074..d4e0d1da84 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos; @@ -26,8 +27,6 @@ public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); - -// void onEntityProfileUpdate(TransportProtos.CalculatedFieldEntityProfileUpdateMsgProto proto, TbCallback callback); + void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index 4982445735..87f1d08a84 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -25,7 +25,7 @@ public class CalculatedFieldResult { private String type; private AttributeScope scope; - private Map resultMap; + private Map resultMap; public CalculatedFieldResult() { } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 250496830c..c83bd7a71a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; @@ -30,6 +31,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; @@ -90,6 +92,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final TimeseriesService timeseriesService; private final RocksDBService rocksDBService; private final TbClusterService clusterService; + private final TbelInvokeService tbelInvokeService; private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; @@ -201,7 +204,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { CalculatedField calculatedField = calculatedFields.computeIfAbsent(calculatedFieldId, id -> calculatedFieldService.findById(tenantId, id)); updateOrInitializeState(calculatedField, calculatedField.getEntityId(), updatedTelemetry); @@ -266,13 +269,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { Map arguments = calculatedField.getConfiguration().getArguments(); - Map argumentValues = new HashMap<>(); + Map argumentValues = new HashMap<>(); AtomicInteger remaining = new AtomicInteger(arguments.size()); arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { @Override public void onSuccess(Optional result) { - String value = result.map(KvEntry::getValueAsString).orElse(argument.getDefaultValue()); - argumentValues.put(key, value); + // todo: should be rewritten implementation for default value + argumentValues.put(key, result.orElse(null)); if (remaining.decrementAndGet() == 0) { updateOrInitializeState(calculatedField, entityId, argumentValues); } @@ -300,7 +303,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }; } - private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { + private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(calculatedField.getUuidId(), entityId.getId()); CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(ctxId, null)); @@ -314,10 +317,21 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.put(ctxId, calculatedFieldCtx); rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); - CalculatedFieldResult result = state.performCalculation(calculatedField.getConfiguration()); - if (result != null) { - pushMsgToRuleEngine(calculatedField.getTenantId(), calculatedField.getEntityId(), result); - } + ListenableFuture resultFuture = state.performCalculation(calculatedField.getTenantId(), calculatedField.getConfiguration(), tbelInvokeService); + Futures.addCallback(resultFuture, new FutureCallback<>() { + @Override + public void onSuccess(CalculatedFieldResult result) { + if (result != null) { + pushMsgToRuleEngine(calculatedField.getTenantId(), calculatedField.getEntityId(), result); + } + } + + @Override + public void onFailure(Throwable t) { + log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedField.getId(), entityId, t); + } + }, MoreExecutors.directExecutor()); + } private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { @@ -325,8 +339,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas String type = calculatedFieldResult.getType(); TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; - ObjectNode jsonNodes = createJsonPayload(calculatedFieldResult); - TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(jsonNodes)); + ObjectNode payload = createJsonPayload(calculatedFieldResult); + TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(payload)); clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); } catch (Exception e) { log.warn("[{}] Failed to push message to rule engine. CalculatedFieldResult: {}", originatorId, calculatedFieldResult, e); @@ -334,9 +348,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { - ObjectNode jsonNodes = JacksonUtil.newObjectNode(); - calculatedFieldResult.getResultMap().forEach(jsonNodes::put); - return jsonNodes; + ObjectNode payload = JacksonUtil.newObjectNode(); + Map resultMap = calculatedFieldResult.getResultMap(); + resultMap.forEach((k, v) -> payload.set(k, JacksonUtil.convertValue(v, JsonNode.class))); + return payload; } private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java new file mode 100644 index 0000000000..6b54536019 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.kv.KvEntry; + +import java.util.Map; + +public interface CalculatedFieldScriptEngine { + + ListenableFuture executeScriptAsync(Map arguments); + + void destroy(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index c25a6960ac..dffcf09820 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -18,9 +18,13 @@ package org.thingsboard.server.service.cf.ctx.state; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.Map; @@ -39,12 +43,12 @@ public interface CalculatedFieldState { @JsonIgnore CalculatedFieldType getType(); - default boolean isValid(Map argumentValues, Map arguments) { + default boolean isValid(Map argumentValues, Map arguments) { return argumentValues.keySet().containsAll(arguments.keySet()); } - void initState(Map argumentValues); + void initState(Map argumentValues); - CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration); + ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java new file mode 100644 index 0000000000..7e8376be8e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.ScriptType; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; + +import javax.script.ScriptException; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +@Slf4j +public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEngine { + + private final TbelInvokeService tbelInvokeService; + + private final UUID scriptId; + private final TenantId tenantId; + + public CalculatedFieldTbelScriptEngine(TenantId tenantId, TbelInvokeService tbelInvokeService, String script, String... argNames) { + this.tenantId = tenantId; + this.tbelInvokeService = tbelInvokeService; + try { + this.scriptId = this.tbelInvokeService.eval(tenantId, ScriptType.CALCULATED_FIELD_SCRIPT, script, argNames).get(); + } catch (Exception e) { + Throwable t = e; + if (e instanceof ExecutionException) { + t = e.getCause(); + } + throw new IllegalArgumentException("Can't compile script: " + t.getMessage(), t); + } + } + + @Override + public ListenableFuture executeScriptAsync(Map arguments) { + log.trace("execute script async, arguments {}", arguments); + Object[] args = new Object[arguments.size()]; + int index = 0; + for (KvEntry entry : arguments.values()) { + switch (entry.getDataType()) { + case BOOLEAN -> args[index] = entry.getBooleanValue().orElse(null); + case DOUBLE -> args[index] = entry.getDoubleValue().orElse(null); + case LONG -> args[index] = entry.getLongValue().orElse(null); + case JSON -> args[index] = entry.getJsonValue().map(JacksonUtil::toJsonNode).orElse(null); + default -> args[index] = entry.getValueAsString(); + } + index++; + } + return Futures.transformAsync(tbelInvokeService.invokeScript(tenantId, null, this.scriptId, args), + o -> { + try { + return Futures.immediateFuture(o); + } catch (Exception e) { + if (e.getCause() instanceof ScriptException) { + return Futures.immediateFailedFuture(e.getCause()); + } else if (e.getCause() instanceof RuntimeException) { + return Futures.immediateFailedFuture(new ScriptException(e.getCause().getMessage())); + } else { + return Futures.immediateFailedFuture(new ScriptException(e)); + } + } + }, MoreExecutors.directExecutor()); + } + + @Override + public void destroy() { + tbelInvokeService.release(this.scriptId); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 238e8005f2..363c237df5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -15,10 +15,19 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -28,7 +37,10 @@ import java.util.Map; @Slf4j public class ScriptCalculatedFieldState implements CalculatedFieldState { - private Map arguments = new HashMap<>(); + @JsonIgnore + private CalculatedFieldScriptEngine calculatedFieldScriptEngine; + + private Map arguments = new HashMap<>(); public ScriptCalculatedFieldState() { } @@ -39,7 +51,7 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { } @Override - public void initState(Map argumentValues) { + public void initState(Map argumentValues) { if (arguments == null) { this.arguments = new HashMap<>(); } @@ -47,9 +59,46 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { } @Override - public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { - // TODO: implement - return null; + public ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { + if (tbelInvokeService == null) { + throw new IllegalArgumentException("TBEL script engine is disabled!"); + } + + if (calculatedFieldScriptEngine == null) { + initEngine(tenantId, calculatedFieldConfiguration, tbelInvokeService); + } + + ListenableFuture resultFuture = calculatedFieldScriptEngine.executeScriptAsync(arguments); + + return Futures.transform(resultFuture, result -> { + Output output = calculatedFieldConfiguration.getOutput(); + Map resultMap = new HashMap<>(); + + if (result instanceof Map) { + Map map = JacksonUtil.convertValue(result, Map.class); + if (map != null) { + resultMap.putAll(map); + } + } else { + resultMap.put(output.getName(), JacksonUtil.convertValue(result, Object.class)); + } + + CalculatedFieldResult calculatedFieldResult = new CalculatedFieldResult(); + calculatedFieldResult.setType(output.getType()); + calculatedFieldResult.setScope(output.getScope()); + calculatedFieldResult.setResultMap(resultMap); + + return calculatedFieldResult; + }, MoreExecutors.directExecutor()); + } + + private void initEngine(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { + calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( + tenantId, + tbelInvokeService, + calculatedFieldConfiguration.getOutput().getExpression(), + arguments.keySet().toArray(new String[0]) + ); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index e984e300c6..725da6c7a7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -15,13 +15,18 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; import net.objecthunter.exp4j.Expression; import net.objecthunter.exp4j.ExpressionBuilder; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -30,8 +35,7 @@ import java.util.Map; @Data public class SimpleCalculatedFieldState implements CalculatedFieldState { - // TODO: use value object(TsKv) instead of string - private Map arguments; + private Map arguments; @Override public CalculatedFieldType getType() { @@ -39,7 +43,7 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { } @Override - public void initState(Map argumentValues) { + public void initState(Map argumentValues) { if (arguments == null) { arguments = new HashMap<>(); } @@ -47,7 +51,7 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { } @Override - public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { + public ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { Output output = calculatedFieldConfiguration.getOutput(); Map arguments = calculatedFieldConfiguration.getArguments(); @@ -64,19 +68,17 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { customExpression.set(expr); } Map variables = new HashMap<>(); - this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v))); + this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValueAsString()))); expr.setVariables(variables); - String expressionResult = String.valueOf(expr.evaluate()); + double expressionResult = expr.evaluate(); result.setType(output.getType()); result.setScope(output.getScope()); result.setResultMap(Map.of(output.getName(), expressionResult)); - return result; + return Futures.immediateFuture(result); } - return null; - // TODO: handle what happens when not valid } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index ac769bee9e..76c1383f6a 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -69,6 +69,7 @@ import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.stream.Collectors; /** * Created by ashvayka on 27.03.18. @@ -207,32 +208,28 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); Map attributes = link.getConfiguration().getAttributes(); Map timeSeries = link.getConfiguration().getTimeSeries(); - List filteredTelemetry = telemetry.stream() + Map updatedTelemetry = telemetry.stream() .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) - .toList(); - - - Map updatedTelemetry = new HashMap<>(); - for (KvEntry telemetryEntry : filteredTelemetry) { - String key = telemetryEntry.getKey(); - if (telemetryEntry instanceof AttributeKvEntry) { - for (Map.Entry attribute : attributes.entrySet()) { - if (telemetryEntry.getKey().equals(attribute.getValue())) { - key = attribute.getKey(); - break; - } - } - } - if (telemetryEntry instanceof TsKvEntry) { - for (Map.Entry timeSeriesEntry : timeSeries.entrySet()) { - if (telemetryEntry.getKey().equals(timeSeriesEntry.getValue())) { - key = timeSeriesEntry.getKey(); - break; - } - } - } - updatedTelemetry.put(key, telemetryEntry.getValueAsString()); - } + .collect(Collectors.toMap( + entry -> { + if (entry instanceof AttributeKvEntry) { + return attributes.entrySet().stream() + .filter(attr -> attr.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } else if (entry instanceof TsKvEntry) { + return timeSeries.entrySet().stream() + .filter(ts -> ts.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } + return entry.getKey(); + }, + entry -> entry, + (v1, v2) -> v1 + )); if (!updatedTelemetry.isEmpty()) { calculatedFieldExecutionService.onTelemetryUpdate(tenantId, calculatedFieldId, updatedTelemetry); diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java index cdcdf815d0..7f8c513957 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java @@ -16,5 +16,5 @@ package org.thingsboard.script.api; public enum ScriptType { - RULE_NODE_SCRIPT + RULE_NODE_SCRIPT, CALCULATED_FIELD_SCRIPT } From 6ed4abb9038eb424c129965d99fae7ed828c633a Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 21 Nov 2024 17:39:16 +0200 Subject: [PATCH 04/21] separated expression from output --- ...efaultCalculatedFieldExecutionService.java | 4 +-- .../ctx/state/ScriptCalculatedFieldState.java | 2 +- .../ctx/state/SimpleCalculatedFieldState.java | 2 +- .../CalculatedFieldControllerTest.java | 8 +++--- .../BaseCalculatedFieldConfiguration.java | 27 +++++++++++++++---- .../CalculatedFieldConfiguration.java | 2 ++ .../common/data/cf/configuration/Output.java | 1 - ...efaultNativeCalculatedFieldRepository.java | 5 +++- .../server/dao/service/AssetServiceTest.java | 6 +++-- .../service/CalculatedFieldServiceTest.java | 8 +++--- .../dao/service/CustomerServiceTest.java | 6 +++-- .../server/dao/service/DeviceServiceTest.java | 6 +++-- 12 files changed, 54 insertions(+), 23 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index c83bd7a71a..6aa9997d1f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -253,9 +253,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); - boolean outputExpressionChanged = !oldConfig.getOutput().getExpression().equals(newConfig.getOutput().getExpression()); + boolean expressionChanged = !oldConfig.getExpression().equals(newConfig.getExpression()); - return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputExpressionChanged; + return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; } private void fetchCalculatedFields() { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 363c237df5..cbee8dc902 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -96,7 +96,7 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( tenantId, tbelInvokeService, - calculatedFieldConfiguration.getOutput().getExpression(), + calculatedFieldConfiguration.getExpression(), arguments.keySet().toArray(new String[0]) ); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 725da6c7a7..08c4741de5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -57,7 +57,7 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { if (isValid(this.arguments, arguments)) { CalculatedFieldResult result = new CalculatedFieldResult(); - String expression = output.getExpression(); + String expression = calculatedFieldConfiguration.getExpression(); ThreadLocal customExpression = new ThreadLocal<>(); var expr = customExpression.get(); if (expr == null) { diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 314dc2bdba..77ca268d12 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -21,10 +21,10 @@ import org.junit.Test; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; @@ -145,9 +145,11 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { config.setArguments(Map.of("T", argument)); + config.setExpression("T - (100 - H) / 5"); + Output output = new Output(); + output.setName("output"); output.setType("TIME_SERIES"); - output.setExpression("T - (100 - H) / 5"); config.setOutput(output); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index 7692d792f8..55a43a00ce 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -40,6 +40,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel private final ObjectMapper mapper = new ObjectMapper(); protected Map arguments; + protected String expression; protected Output output; public BaseCalculatedFieldConfiguration() { @@ -48,6 +49,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel public BaseCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { BaseCalculatedFieldConfiguration calculatedFieldConfig = toCalculatedFieldConfig(config, entityType, entityId); this.arguments = calculatedFieldConfig.getArguments(); + this.expression = calculatedFieldConfig.getExpression(); this.output = calculatedFieldConfig.getOutput(); } @@ -101,12 +103,17 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argumentNode.put("defaultValue", argument.getDefaultValue()); }); + if (expression != null) { + configNode.put("expression", expression); + } + if (output != null) { ObjectNode outputNode = configNode.putObject("output"); outputNode.put("name", output.getName()); outputNode.put("type", output.getType()); - outputNode.put("scope", String.valueOf(output.getScope())); - outputNode.put("expression", output.getExpression()); + if (output.getScope() != null) { + outputNode.put("scope", String.valueOf(output.getScope())); + } } return configNode; @@ -133,20 +140,30 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel } argument.setKey(argumentNode.get("key").asText()); argument.setType(argumentNode.get("type").asText()); - argument.setScope(AttributeScope.valueOf(argumentNode.get("scope").asText())); + JsonNode scope = argumentNode.get("scope"); + if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { + argument.setScope(AttributeScope.valueOf(scope.asText())); + } argument.setDefaultValue(argumentNode.get("defaultValue").asText()); arguments.put(key, argument); }); } this.setArguments(arguments); + JsonNode expressionNode = config.get("expression"); + if (expressionNode != null && expressionNode.isTextual()) { + this.setExpression(expressionNode.asText()); + } + JsonNode outputNode = config.get("output"); if (outputNode != null) { Output output = new Output(); output.setName(outputNode.get("name").asText()); output.setType(outputNode.get("type").asText()); - output.setScope(AttributeScope.valueOf(outputNode.get("scope").asText())); - output.setExpression(outputNode.get("expression").asText()); + JsonNode scope = outputNode.get("scope"); + if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { + output.setScope(AttributeScope.valueOf(scope.asText())); + } this.setOutput(output); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 155015028f..5c428bd628 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -44,6 +44,8 @@ public interface CalculatedFieldConfiguration { Map getArguments(); + String getExpression(); + Output getOutput(); @JsonIgnore diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java index 683e372ebc..46257d1ccc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java @@ -24,6 +24,5 @@ public class Output { private String name; private String type; private AttributeScope scope; - private String expression; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index eebca14b6e..fc40f72c93 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -25,10 +25,11 @@ import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; @@ -138,6 +139,8 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF switch (type) { case SIMPLE: return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + case SCRIPT: + return new ScriptCalculatedFieldConfiguration(config, entityType, entityId); default: throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 2012090264..87f2cb1f45 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -30,9 +30,9 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; @@ -889,9 +889,11 @@ public class AssetServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); + config.setExpression("T - (100 - H) / 5"); + Output output = new Output(); + output.setName("output"); output.setType("TIME_SERIES"); - output.setExpression("T - (100 - H) / 5"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 2bdb1b8897..77ed026b1d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -23,10 +23,10 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -158,9 +158,11 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); + config.setExpression("T - (100 - H) / 5"); + Output output = new Output(); + output.setName("output"); output.setType("TIME_SERIES"); - output.setExpression("T - (100 - H) / 5"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index b58f739462..94c8440057 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -31,9 +31,9 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; @@ -384,9 +384,11 @@ public class CustomerServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); + config.setExpression("T - (100 - H) / 5"); + Output output = new Output(); + output.setName("output"); output.setType("TIME_SERIES"); - output.setExpression("T - (100 - H) / 5"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 38bd21170a..d5394a3494 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -39,9 +39,9 @@ import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; @@ -1227,9 +1227,11 @@ public class DeviceServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); + config.setExpression("T - (100 - H) / 5"); + Output output = new Output(); + output.setName("output"); output.setType("TIME_SERIES"); - output.setExpression("T - (100 - H) / 5"); config.setOutput(output); From 31103e90e3e3cb5f440823872f2a0c85cdac6b10 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 22 Nov 2024 17:17:59 +0200 Subject: [PATCH 05/21] added implementation to handle update entity profiles events --- .../cf/CalculatedFieldExecutionService.java | 2 + ...efaultCalculatedFieldExecutionService.java | 74 ++++++++++++++++--- .../CalculatedFieldTbelScriptEngine.java | 14 +--- .../ctx/state/ScriptCalculatedFieldState.java | 13 +--- .../entitiy/EntityStateSourcingListener.java | 31 ++++---- .../queue/DefaultTbClusterService.java | 44 ++++++++++- .../queue/DefaultTbCoreConsumerService.java | 19 ++++- .../server/cluster/TbClusterService.java | 3 + .../server/dao/cf/CalculatedFieldService.java | 2 + common/proto/src/main/proto/queue.proto | 34 ++++++--- .../server/dao/asset/BaseAssetService.java | 4 +- .../dao/cf/BaseCalculatedFieldService.java | 14 +++- .../server/dao/cf/CalculatedFieldDao.java | 3 + .../dao/sql/cf/CalculatedFieldRepository.java | 3 + .../dao/sql/cf/JpaCalculatedFieldDao.java | 6 ++ 15 files changed, 202 insertions(+), 64 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index d4e0d1da84..6b7b12655f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -29,4 +29,6 @@ public interface CalculatedFieldExecutionService { void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); + void onEntityTypeChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 6aa9997d1f..9edea50a05 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -27,12 +27,14 @@ import jakarta.annotation.PreDestroy; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.math.NumberUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -44,8 +46,14 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; @@ -210,7 +218,38 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas updateOrInitializeState(calculatedField, calculatedField.getEntityId(), updatedTelemetry); log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { - log.trace("Failed to update time series for calculatedFieldId: [{}]", calculatedFieldId, e); + log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); + } + } + + @Override + public void onEntityTypeChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); + EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); + + log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + + List cfIdsOfOldProfile = calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId); + cfIdsOfOldProfile.forEach(id -> states.remove(new CalculatedFieldCtxId(id.getId(), entityId.getId()))); + List ctxIdsToDelete = cfIdsOfOldProfile.stream().map(cfId -> JacksonUtil.writeValueAsString(new CalculatedFieldCtxId(cfId.getId(), entityId.getId()))).toList(); + rocksDBService.deleteAll(ctxIdsToDelete); + + calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) + .forEach(cfId -> { + CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(cfId.getId(), entityId.getId()); + states.remove(ctxId); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + }); + + calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, newProfileId) + .stream() + .map(cfId -> calculatedFields.computeIfAbsent(cfId, id -> calculatedFieldService.findById(tenantId, id))) + .forEach(cf -> initializeStateForEntity(tenantId, cf, entityId, callback)); + } catch (Exception e) { + log.trace("Failed to process entity type update msg: [{}]", proto, e); } } @@ -271,10 +310,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Map arguments = calculatedField.getConfiguration().getArguments(); Map argumentValues = new HashMap<>(); AtomicInteger remaining = new AtomicInteger(arguments.size()); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument, entityId), new FutureCallback<>() { @Override public void onSuccess(Optional result) { - // todo: should be rewritten implementation for default value argumentValues.put(key, result.orElse(null)); if (remaining.decrementAndGet() == 0) { updateOrInitializeState(calculatedField, entityId, argumentValues); @@ -289,20 +327,38 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor)); } - private ListenableFuture> fetchArgumentValue(TenantId tenantId, Argument argument) { + private ListenableFuture> fetchArgumentValue(TenantId tenantId, Argument argument, EntityId targetEntityId) { + EntityId argumentEntityId = argument.getEntityId(); + EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; return switch (argument.getType()) { case "ATTRIBUTES" -> Futures.transform( - attributesService.find(tenantId, argument.getEntityId(), argument.getScope(), argument.getKey()), - result -> result.map(entry -> (KvEntry) entry), + attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), + result -> result.or(() -> Optional.of( + new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)) + )), MoreExecutors.directExecutor()); case "TIME_SERIES" -> Futures.transform( - timeseriesService.findLatest(tenantId, argument.getEntityId(), argument.getKey()), - result -> result.map(entry -> (KvEntry) entry), + timeseriesService.findLatest(tenantId, entityId, argument.getKey()), + result -> result.or(() -> Optional.of( + new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)) + )), MoreExecutors.directExecutor()); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; } + private KvEntry createDefaultKvEntry(Argument argument) { + String key = argument.getKey(); + String defaultValue = argument.getDefaultValue(); + if (NumberUtils.isParsable(defaultValue)) { + return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); + } + if ("true".equalsIgnoreCase(defaultValue) || "false".equalsIgnoreCase(defaultValue)) { + return new BooleanDataEntry(key, Boolean.parseBoolean(defaultValue)); + } + return new StringDataEntry(key, defaultValue); + } + private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(calculatedField.getUuidId(), entityId.getId()); CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(ctxId, null)); @@ -322,7 +378,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onSuccess(CalculatedFieldResult result) { if (result != null) { - pushMsgToRuleEngine(calculatedField.getTenantId(), calculatedField.getEntityId(), result); + pushMsgToRuleEngine(calculatedField.getTenantId(), entityId, result); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java index 7e8376be8e..5cc58b2a95 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -19,7 +19,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.ScriptType; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.id.TenantId; @@ -55,18 +54,7 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng @Override public ListenableFuture executeScriptAsync(Map arguments) { log.trace("execute script async, arguments {}", arguments); - Object[] args = new Object[arguments.size()]; - int index = 0; - for (KvEntry entry : arguments.values()) { - switch (entry.getDataType()) { - case BOOLEAN -> args[index] = entry.getBooleanValue().orElse(null); - case DOUBLE -> args[index] = entry.getDoubleValue().orElse(null); - case LONG -> args[index] = entry.getLongValue().orElse(null); - case JSON -> args[index] = entry.getJsonValue().map(JacksonUtil::toJsonNode).orElse(null); - default -> args[index] = entry.getValueAsString(); - } - index++; - } + Object[] args = arguments.values().stream().map(KvEntry::getValue).toArray(); return Futures.transformAsync(tbelInvokeService.invokeScript(tenantId, null, this.scriptId, args), o -> { try { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index cbee8dc902..5c984a8d16 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -72,16 +72,9 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { return Futures.transform(resultFuture, result -> { Output output = calculatedFieldConfiguration.getOutput(); - Map resultMap = new HashMap<>(); - - if (result instanceof Map) { - Map map = JacksonUtil.convertValue(result, Map.class); - if (map != null) { - resultMap.putAll(map); - } - } else { - resultMap.put(output.getName(), JacksonUtil.convertValue(result, Object.class)); - } + Map resultMap = result instanceof Map + ? JacksonUtil.convertValue(result, Map.class) + : new HashMap<>(); CalculatedFieldResult calculatedFieldResult = new CalculatedFieldResult(); calculatedFieldResult.setType(output.getType()); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 1b541bcd5d..154f5f4833 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.Edge; @@ -51,8 +52,6 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceCredentialsUpdateNotificationMsg; -import org.thingsboard.server.dao.cf.CalculatedFieldService; -import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; @@ -67,8 +66,6 @@ public class EntityStateSourcingListener { private final TbClusterService tbClusterService; private final TenantService tenantService; - private final CalculatedFieldService calculatedFieldService; - private final DeviceProfileService deviceProfileService; @PostConstruct public void init() { @@ -88,7 +85,10 @@ public class EntityStateSourcingListener { ComponentLifecycleEvent lifecycleEvent = isCreated ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED; switch (entityType) { - case ASSET, ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE -> { + case ASSET -> { + onAssetUpdate(event.getEntity(), event.getOldEntity()); + } + case ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, lifecycleEvent); } case RULE_CHAIN -> { @@ -106,7 +106,7 @@ public class EntityStateSourcingListener { onTenantProfileUpdate(tenantProfile, lifecycleEvent); } case DEVICE -> { - onDeviceUpdate(tenantId, event.getEntity(), event.getOldEntity()); + onDeviceUpdate(event.getEntity(), event.getOldEntity()); } case DEVICE_PROFILE -> { DeviceProfile deviceProfile = (DeviceProfile) event.getEntity(); @@ -245,23 +245,24 @@ public class EntityStateSourcingListener { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); } - private void onDeviceUpdate(TenantId tenantId, Object entity, Object oldEntity) { + private void onDeviceUpdate(Object entity, Object oldEntity) { Device device = (Device) entity; Device oldDevice = null; if (oldEntity instanceof Device) { oldDevice = (Device) oldEntity; - // TODO: move verification of device type to cluster service - if (!oldDevice.getType().equals(device.getType())) { - DeviceProfile profile = deviceProfileService.findDeviceProfileByName(tenantId, device.getType()); - boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profile.getId()); - if (cfExistsByProfile) { - // TODO: send device type updated msg to core - } - } } tbClusterService.onDeviceUpdated(device, oldDevice); } + private void onAssetUpdate(Object entity, Object oldEntity) { + Asset asset = (Asset) entity; + Asset oldAsset = null; + if (oldEntity instanceof Asset) { + oldAsset = (Asset) oldEntity; + } + tbClusterService.onAssetUpdated(asset, oldAsset); + } + private void onEdgeEvent(TenantId tenantId, EntityId entityId, Object entity, ComponentLifecycleEvent lifecycleEvent) { if (entity instanceof Edge) { tbClusterService.onEdgeStateChangeEvent(new ComponentLifecycleMsg(tenantId, entityId, lifecycleEvent)); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index f7c8230716..58a16d9c0a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -68,6 +68,7 @@ import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; @@ -149,6 +150,7 @@ public class DefaultTbClusterService implements TbClusterService { private final GatewayNotificationsService gatewayNotificationsService; private final EdgeService edgeService; private final TbTransactionalCache edgeIdServiceIdCache; + private final CalculatedFieldService calculatedFieldService; @Override public void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback) { @@ -609,7 +611,11 @@ public class DefaultTbClusterService implements TbClusterService { if (deviceNameChanged) { gatewayNotificationsService.onDeviceUpdated(device, old); } - if (deviceNameChanged || !device.getType().equals(old.getType())) { + boolean deviceTypeChanged = !device.getType().equals(old.getType()); + if (deviceTypeChanged) { + handleProfileChange(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); + } + if (deviceNameChanged || deviceTypeChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); } } @@ -618,6 +624,26 @@ public class DefaultTbClusterService implements TbClusterService { otaPackageStateService.update(device, old); } + @Override + public void onAssetUpdated(Asset asset, Asset old) { + var created = old == null; + broadcastEntityChangeToTransport(asset.getTenantId(), asset.getId(), asset, null); + if (old != null) { + boolean assetTypeChanged = !asset.getType().equals(old.getType()); + if (assetTypeChanged) { + handleProfileChange(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); + } + } + broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + } + + private void handleProfileChange(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { + boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, oldProfileId); + if (cfExistsByProfile) { + sendEntityTypeUpdatedEvent(tenantId, entityId, oldProfileId, newProfileId); + } + } + @Override public void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId originatorEdgeId) { if (!edgesEnabled) { @@ -775,4 +801,20 @@ public class DefaultTbClusterService implements TbClusterService { pushMsgToCore(tenantId, calculatedFieldId, ToCoreMsg.newBuilder().setCalculatedFieldMsg(msg).build(), null); } + private void sendEntityTypeUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { + TransportProtos.EntityProfileUpdateMsgProto.Builder builder = TransportProtos.EntityProfileUpdateMsgProto.newBuilder(); + builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + builder.setEntityType(entityId.getEntityType().name()); + builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + builder.setEntityProfileType(newProfileId.getEntityType().name()); + builder.setOldProfileIdMSB(oldProfileId.getId().getMostSignificantBits()); + builder.setOldProfileIdLSB(oldProfileId.getId().getLeastSignificantBits()); + builder.setNewProfileIdMSB(newProfileId.getId().getMostSignificantBits()); + builder.setNewProfileIdLSB(newProfileId.getId().getLeastSignificantBits()); + TransportProtos.EntityProfileUpdateMsgProto msg = builder.build(); + pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityProfileUpdateMsg(msg).build(), null); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 2d3323e097..3caaab6613 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.NotificationRequestId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; @@ -157,6 +158,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> firmwareStatesConsumer; private volatile ListeningExecutorService deviceActivityEventsExecutor; + private volatile ListeningExecutorService calculatedFieldsExecutor; public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, @@ -202,6 +204,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig>builder() .queueKey(new QueueKey(ServiceType.TB_CORE)) @@ -315,6 +318,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = deviceActivityEventsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { @@ -677,6 +682,18 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityTypeChanged(profileUpdateMsg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process device type updated message for device [{}]", tenantId.getId(), entityId.getId(), t); + callback.onFailure(t); + }); + } + private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index f173005107..131b50a52c 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; @@ -97,6 +98,8 @@ public interface TbClusterService extends TbQueueClusterService { void onDeviceAssignedToTenant(TenantId oldTenantId, Device device); + void onAssetUpdated(Asset asset, Asset old); + void onResourceChange(TbResourceInfo resource, TbQueueCallback callback); void onResourceDeleted(TbResourceInfo resource, TbQueueCallback callback); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index e44ff0ba22..1e64fdac60 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -36,6 +36,8 @@ public interface CalculatedFieldService extends EntityDaoService { ListenableFuture findCalculatedFieldByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId); + List findAllCalculatedFields(); PageData findAllCalculatedFields(PageLink pageLink); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 95e4fa8601..bf49a68c80 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -771,6 +771,29 @@ message DeviceInactivityProto { int64 lastInactivityTime = 5; } +message CalculatedFieldMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; + bool added = 5; + bool updated = 6; + bool deleted = 7; +} + +message EntityProfileUpdateMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + string entityType = 3; + int64 entityIdMSB = 4; + int64 entityIdLSB = 5; + string entityProfileType = 6; + int64 oldProfileIdMSB = 7; + int64 oldProfileIdLSB = 8; + int64 newProfileIdMSB = 9; + int64 newProfileIdLSB = 10; +} + //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; @@ -1267,16 +1290,6 @@ message ToDeviceActorNotificationMsgProto { DeviceDeleteMsgProto deviceDeleteMsg = 8; } -message CalculatedFieldMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 calculatedFieldIdMSB = 3; - int64 calculatedFieldIdLSB = 4; - bool added = 5; - bool updated = 6; - bool deleted = 7; -} - /** TB Core to Version Control Service */ @@ -1513,6 +1526,7 @@ message ToCoreMsg { DeviceDisconnectProto deviceDisconnectMsg = 51; DeviceInactivityProto deviceInactivityMsg = 52; CalculatedFieldMsgProto calculatedFieldMsg = 53; + EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index 2135792174..7242cfde68 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -177,8 +177,8 @@ public class BaseAssetService extends AbstractCachedEntityService findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing findCalculatedFieldIdsByEntityId [{}]", entityId); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); + return calculatedFieldDao.findCalculatedFieldIdsByEntityId(tenantId, entityId); + } + @Override public List findAllCalculatedFields() { log.trace("Executing findAll"); @@ -133,7 +141,7 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements public int deleteAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { log.trace("Executing deleteAllCalculatedFieldsByEntityId, tenantId [{}], entityId [{}]", tenantId, entityId); validateId(tenantId, id -> INCORRECT_TENANT_ID + id); - validateId(entityId.getId(), id -> "Incorrect entityId " + id); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); List calculatedFields = calculatedFieldDao.removeAllByEntityId(tenantId, entityId); return calculatedFields.size(); } @@ -212,7 +220,7 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements @Override public boolean existsCalculatedFieldByEntityId(TenantId tenantId, EntityId entityId) { return calculatedFieldDao.existsByEntityId(tenantId, entityId); - }; + } @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index a6b7c2dea1..5b3bcc2750 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.cf; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -28,6 +29,8 @@ public interface CalculatedFieldDao extends Dao { List findAllByTenantId(TenantId tenantId); + List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId); + List findAll(); PageData findAll(PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index 333057e8c5..9aa0aee428 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; import java.util.List; @@ -25,6 +26,8 @@ public interface CalculatedFieldRepository extends JpaRepository findCalculatedFieldIdsByTenantIdAndEntityId(UUID tenantId, UUID entityId); + List findAllByTenantId(UUID tenantId); List removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 737a089a15..e3762f6157 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -22,6 +22,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -49,6 +50,11 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId) { + return calculatedFieldRepository.findCalculatedFieldIdsByTenantIdAndEntityId(tenantId.getId(), entityId.getId()); + } + @Override public List findAll() { return DaoUtil.convertDataList(calculatedFieldRepository.findAll()); From 2c7c6f0c5edfe140a1ce0e528824ec58c48b5647 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 25 Nov 2024 08:57:09 +0200 Subject: [PATCH 06/21] fixed tests for cluster service --- .../server/service/queue/DefaultTbClusterServiceTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java b/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java index 25fe589a08..cb61980cf4 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java @@ -44,6 +44,7 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueCallback; @@ -102,6 +103,8 @@ public class DefaultTbClusterServiceTest { protected TbRuleEngineProducerService ruleEngineProducerService; @MockBean protected TbTransactionalCache edgeCache; + @MockBean + protected CalculatedFieldService calculatedFieldService; @SpyBean protected TopicService topicService; From c6d91c4ce870886e4af404246020dba15e089dd7 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 25 Nov 2024 17:28:44 +0200 Subject: [PATCH 07/21] added last records type of cf --- ...efaultCalculatedFieldExecutionService.java | 84 ++++++++++++++--- .../service/cf/ctx/state/ArgumentEntry.java | 30 ++++++ .../cf/ctx/state/CalculatedFieldState.java | 10 +- .../cf/ctx/state/CalculationContext.java | 35 +++++++ .../LastRecordsCalculatedFieldState.java | 91 +++++++++++++++++++ .../ctx/state/ScriptCalculatedFieldState.java | 17 ++-- .../ctx/state/SimpleCalculatedFieldState.java | 13 ++- .../common/data/cf/CalculatedFieldType.java | 2 +- .../data/cf/configuration/Argument.java | 1 + .../BaseCalculatedFieldConfiguration.java | 16 +++- .../CalculatedFieldConfiguration.java | 3 +- ...stRecordsCalculatedFieldConfiguration.java | 39 ++++++++ .../dao/model/sql/CalculatedFieldEntity.java | 16 ++-- ...efaultNativeCalculatedFieldRepository.java | 14 ++- 14 files changed, 323 insertions(+), 48 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 9edea50a05..a985f093c9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -48,12 +48,16 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; @@ -69,7 +73,10 @@ import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtxId; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.CalculationContext; +import org.thingsboard.server.service.cf.ctx.state.LastRecordsCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; @@ -84,6 +91,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; @@ -109,6 +117,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); + private static final int MAX_LAST_RECORDS_VALUE = 1024; + @Value("${calculatedField.initFetchPackSize:50000}") @Getter private int initFetchPackSize; @@ -215,7 +225,19 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { CalculatedField calculatedField = calculatedFields.computeIfAbsent(calculatedFieldId, id -> calculatedFieldService.findById(tenantId, id)); - updateOrInitializeState(calculatedField, calculatedField.getEntityId(), updatedTelemetry); + Map argumentValues = updatedTelemetry.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> { + ArgumentEntry argumentEntry = new ArgumentEntry(); + argumentEntry.setKvEntry(entry.getValue()); + if (entry.getValue() instanceof TsKvEntry) { + argumentEntry.setKvEntries(List.of((TsKvEntry) entry.getValue())); + } + return argumentEntry; + } + )); + updateOrInitializeState(calculatedField, calculatedField.getEntityId(), argumentValues); log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); @@ -308,12 +330,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { Map arguments = calculatedField.getConfiguration().getArguments(); - Map argumentValues = new HashMap<>(); + Map argumentValues = new HashMap<>(); AtomicInteger remaining = new AtomicInteger(arguments.size()); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument, entityId), new FutureCallback<>() { + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedField, argument), new FutureCallback<>() { @Override - public void onSuccess(Optional result) { - argumentValues.put(key, result.orElse(null)); + public void onSuccess(ArgumentEntry result) { + argumentValues.put(key, result); if (remaining.decrementAndGet() == 0) { updateOrInitializeState(calculatedField, entityId, argumentValues); } @@ -327,10 +349,37 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor)); } - private ListenableFuture> fetchArgumentValue(TenantId tenantId, Argument argument, EntityId targetEntityId) { + private ListenableFuture fetchArgumentValue(CalculatedField calculatedField, Argument argument) { + TenantId tenantId = calculatedField.getTenantId(); + EntityId cfEntityId = calculatedField.getEntityId(); EntityId argumentEntityId = argument.getEntityId(); - EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; - return switch (argument.getType()) { + EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) + ? cfEntityId + : argumentEntityId; + if (CalculatedFieldType.LAST_RECORDS.equals(calculatedField.getType())) { + return fetchLastRecords(tenantId, entityId, argument); + } + return fetchKvEntry(tenantId, entityId, argument); + } + + private ListenableFuture fetchLastRecords(TenantId tenantId, EntityId entityId, Argument argument) { + long startTs = Math.max(argument.getStartTs(), 0); + long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); + long endTs = startTs + timeWindow; + int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); + + ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, endTs, 0, limit, Aggregation.NONE); + ListenableFuture> lastRecordsFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); + + return Futures.transform(lastRecordsFuture, lastRecords -> { + ArgumentEntry argumentEntry = new ArgumentEntry(); + argumentEntry.setKvEntries(lastRecords); + return argumentEntry; + }, calculatedFieldExecutor); + } + + private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { + ListenableFuture> kvEntryFuture = switch (argument.getType()) { case "ATTRIBUTES" -> Futures.transform( attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), result -> result.or(() -> Optional.of( @@ -342,9 +391,16 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas result -> result.or(() -> Optional.of( new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)) )), - MoreExecutors.directExecutor()); + calculatedFieldExecutor); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; + return Futures.transform(kvEntryFuture, kvEntry -> { + ArgumentEntry argumentEntry = new ArgumentEntry(); + if (kvEntry.isPresent()) { + argumentEntry.setKvEntry(kvEntry.orElse(null)); + } + return argumentEntry; + }, calculatedFieldExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { @@ -359,7 +415,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return new StringDataEntry(key, defaultValue); } - private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { + private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(calculatedField.getUuidId(), entityId.getId()); CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(ctxId, null)); @@ -373,7 +429,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.put(ctxId, calculatedFieldCtx); rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); - ListenableFuture resultFuture = state.performCalculation(calculatedField.getTenantId(), calculatedField.getConfiguration(), tbelInvokeService); + CalculationContext ctx = CalculationContext.builder() + .tenantId(calculatedField.getTenantId()) + .configuration(calculatedField.getConfiguration()) + .tbelInvokeService(tbelInvokeService) + .build(); + ListenableFuture resultFuture = state.performCalculation(ctx); Futures.addCallback(resultFuture, new FutureCallback<>() { @Override public void onSuccess(CalculatedFieldResult result) { @@ -414,6 +475,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return switch (calculatedFieldType) { case SIMPLE -> new SimpleCalculatedFieldState(); case SCRIPT -> new ScriptCalculatedFieldState(); + case LAST_RECORDS -> new LastRecordsCalculatedFieldState(); }; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java new file mode 100644 index 0000000000..29e3417bde --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.Data; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +@Data +public class ArgumentEntry { + + private KvEntry kvEntry; + private List kvEntries; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index dffcf09820..e4c4440921 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -19,11 +19,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -36,7 +33,8 @@ import java.util.Map; ) @JsonSubTypes({ @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE"), - @JsonSubTypes.Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT") + @JsonSubTypes.Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT"), + @JsonSubTypes.Type(value = LastRecordsCalculatedFieldState.class, name = "LAST_RECORDS") }) public interface CalculatedFieldState { @@ -47,8 +45,8 @@ public interface CalculatedFieldState { return argumentValues.keySet().containsAll(arguments.keySet()); } - void initState(Map argumentValues); + void initState(Map argumentValues); - ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService); + ListenableFuture performCalculation(CalculationContext ctx); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java new file mode 100644 index 0000000000..656763ea48 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.Builder; +import lombok.Data; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; + +import java.util.Map; + +@Data +@Builder +public class CalculationContext { + + private TenantId tenantId; + private CalculatedFieldConfiguration configuration; + private TbelInvokeService tbelInvokeService; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java new file mode 100644 index 0000000000..2f26d71f91 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java @@ -0,0 +1,91 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.service.cf.CalculatedFieldResult; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Data +public class LastRecordsCalculatedFieldState implements CalculatedFieldState { + + private Map> arguments; + + public LastRecordsCalculatedFieldState() { + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.LAST_RECORDS; + } + + + @Override + public void initState(Map argumentValues) { + if (arguments == null) { + arguments = new HashMap<>(); + } + argumentValues.forEach((key, argumentEntry) -> { + List tsKvEntryList = arguments.computeIfAbsent(key, k -> new ArrayList<>()); + tsKvEntryList.addAll(argumentEntry.getKvEntries()); + }); + } + + + @Override + public ListenableFuture performCalculation(CalculationContext ctx) { + CalculatedFieldConfiguration configuration = ctx.getConfiguration(); + Map configArguments = configuration.getArguments(); + Output output = configuration.getOutput(); + + Map resultMap = new HashMap<>(); + + arguments.replaceAll((key, entries) -> { + int limit = configArguments.get(key).getLimit(); + List limitedEntries = entries.stream() + .sorted(Comparator.comparingLong(TsKvEntry::getTs).reversed()) + .limit(limit) + .collect(Collectors.toList()); + + Map valueWithTs = limitedEntries.stream() + .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue)); + resultMap.put(key, valueWithTs); + + return limitedEntries; + }); + + CalculatedFieldResult calculatedFieldResult = new CalculatedFieldResult(); + calculatedFieldResult.setType(output.getType()); + calculatedFieldResult.setScope(output.getScope()); + calculatedFieldResult.setResultMap(resultMap); + + return Futures.immediateFuture(calculatedFieldResult); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 5c984a8d16..0e9b00ad7d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -40,7 +40,7 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { @JsonIgnore private CalculatedFieldScriptEngine calculatedFieldScriptEngine; - private Map arguments = new HashMap<>(); + private Map arguments; public ScriptCalculatedFieldState() { } @@ -50,22 +50,27 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { return CalculatedFieldType.SCRIPT; } + @Override - public void initState(Map argumentValues) { + public void initState(Map argumentValues) { if (arguments == null) { - this.arguments = new HashMap<>(); + arguments = new HashMap<>(); } - this.arguments.putAll(argumentValues); + argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); } + @Override - public ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { + public ListenableFuture performCalculation(CalculationContext ctx) { + CalculatedFieldConfiguration calculatedFieldConfiguration = ctx.getConfiguration(); + TbelInvokeService tbelInvokeService = ctx.getTbelInvokeService(); + if (tbelInvokeService == null) { throw new IllegalArgumentException("TBEL script engine is disabled!"); } if (calculatedFieldScriptEngine == null) { - initEngine(tenantId, calculatedFieldConfiguration, tbelInvokeService); + initEngine(ctx.getTenantId(), calculatedFieldConfiguration, tbelInvokeService); } ListenableFuture resultFuture = calculatedFieldScriptEngine.executeScriptAsync(arguments); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 08c4741de5..d3d3b5f636 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -20,12 +20,10 @@ import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; import net.objecthunter.exp4j.Expression; import net.objecthunter.exp4j.ExpressionBuilder; -import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -37,21 +35,26 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { private Map arguments; + public SimpleCalculatedFieldState() { + } + @Override public CalculatedFieldType getType() { return CalculatedFieldType.SIMPLE; } @Override - public void initState(Map argumentValues) { + public void initState(Map argumentValues) { if (arguments == null) { arguments = new HashMap<>(); } - arguments.putAll(argumentValues); + argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); } @Override - public ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { + public ListenableFuture performCalculation(CalculationContext ctx) { + CalculatedFieldConfiguration calculatedFieldConfiguration = ctx.getConfiguration(); + Output output = calculatedFieldConfiguration.getOutput(); Map arguments = calculatedFieldConfiguration.getArguments(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java index 89173b35b9..63b6d8d1dd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -17,6 +17,6 @@ package org.thingsboard.server.common.data.cf; public enum CalculatedFieldType { - SIMPLE, SCRIPT + SIMPLE, SCRIPT, LAST_RECORDS } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index f34f5e9cb7..0d70591a38 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -29,6 +29,7 @@ public class Argument { private String defaultValue; private int limit; + private long startTs; private long timeWindow; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index 55a43a00ce..f7cc53b7cf 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -101,6 +101,9 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argumentNode.put("type", argument.getType()); argumentNode.put("scope", String.valueOf(argument.getScope())); argumentNode.put("defaultValue", argument.getDefaultValue()); + argumentNode.put("limit", String.valueOf(argument.getLimit())); + argumentNode.put("startTs", String.valueOf(argument.getStartTs())); + argumentNode.put("timeWindow", String.valueOf(argument.getTimeWindow())); }); if (expression != null) { @@ -144,7 +147,18 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { argument.setScope(AttributeScope.valueOf(scope.asText())); } - argument.setDefaultValue(argumentNode.get("defaultValue").asText()); + if (argumentNode.hasNonNull("defaultValue")) { + argument.setDefaultValue(argumentNode.get("defaultValue").asText()); + } + if (argumentNode.hasNonNull("limit")) { + argument.setLimit(argumentNode.get("limit").asInt()); + } + if (argumentNode.hasNonNull("startTs")) { + argument.setStartTs(argumentNode.get("startTs").asLong()); + } + if (argumentNode.hasNonNull("timeWindow")) { + argument.setTimeWindow(argumentNode.get("timeWindow").asInt()); + } arguments.put(key, argument); }); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 5c428bd628..15f7a82c40 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -35,7 +35,8 @@ import java.util.UUID; ) @JsonSubTypes({ @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE"), - @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT") + @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT"), + @JsonSubTypes.Type(value = LastRecordsCalculatedFieldConfiguration.class, name = "LAST_RECORDS") }) public interface CalculatedFieldConfiguration { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..c3f5804227 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; + +import java.util.UUID; + +@Data +public class LastRecordsCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + public LastRecordsCalculatedFieldConfiguration() { + } + + public LastRecordsCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + super(config, entityType, entityId); + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.LAST_RECORDS; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 6500d2a1e7..b06676f70b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -24,8 +24,9 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.LastRecordsCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -120,14 +121,11 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem } private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - switch (CalculatedFieldType.valueOf(type)) { - case SIMPLE: - return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); - case SCRIPT: - return new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - default: - throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); - } + return switch (CalculatedFieldType.valueOf(type)) { + case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); + case LAST_RECORDS -> new LastRecordsCalculatedFieldConfiguration(config, entityType, entityId); + }; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index fc40f72c93..2acd4d75c6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.LastRecordsCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -136,14 +137,11 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF } private CalculatedFieldConfiguration readCalculatedFieldConfiguration(CalculatedFieldType type, JsonNode config, EntityType entityType, UUID entityId) { - switch (type) { - case SIMPLE: - return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); - case SCRIPT: - return new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - default: - throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); - } + return switch (type) { + case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); + case LAST_RECORDS -> new LastRecordsCalculatedFieldConfiguration(config, entityType, entityId); + }; } } From 80d9b22068ea44a5fc3d4bada2baa5c704d24246 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 26 Nov 2024 11:31:49 +0200 Subject: [PATCH 08/21] added argument entry interface --- ...efaultCalculatedFieldExecutionService.java | 66 +++++++------------ .../service/cf/ctx/state/ArgumentEntry.java | 19 ++++-- .../ctx/state/BaseCalculatedFieldState.java | 48 ++++++++++++++ .../cf/ctx/state/CalculationContext.java | 6 +- .../service/cf/ctx/state/KvArgumentEntry.java | 31 +++++++++ .../ctx/state/LastRecordsArgumentEntry.java | 33 ++++++++++ .../LastRecordsCalculatedFieldState.java | 21 ++---- .../ctx/state/ScriptCalculatedFieldState.java | 60 ++++++----------- .../ctx/state/SimpleCalculatedFieldState.java | 31 ++------- 9 files changed, 185 insertions(+), 130 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index a985f093c9..322443c848 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -191,19 +191,19 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas switch (entityId.getEntityType()) { case ASSET, DEVICE -> { log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); - initializeStateForEntity(tenantId, cf, entityId, callback); + initializeStateForEntity(cf, entityId, callback); } case ASSET_PROFILE -> { log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, assetId, callback)); + assetIds.forEach(assetId -> initializeStateForEntity(cf, assetId, callback)); } case DEVICE_PROFILE -> { log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, deviceId, callback)); + deviceIds.forEach(deviceId -> initializeStateForEntity(cf, deviceId, callback)); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -226,17 +226,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas try { CalculatedField calculatedField = calculatedFields.computeIfAbsent(calculatedFieldId, id -> calculatedFieldService.findById(tenantId, id)); Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> { - ArgumentEntry argumentEntry = new ArgumentEntry(); - argumentEntry.setKvEntry(entry.getValue()); - if (entry.getValue() instanceof TsKvEntry) { - argumentEntry.setKvEntries(List.of((TsKvEntry) entry.getValue())); - } - return argumentEntry; - } - )); + .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createArgumentEntry(entry.getValue()))); updateOrInitializeState(calculatedField, calculatedField.getEntityId(), argumentValues); log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { @@ -254,11 +244,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - List cfIdsOfOldProfile = calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId); - cfIdsOfOldProfile.forEach(id -> states.remove(new CalculatedFieldCtxId(id.getId(), entityId.getId()))); - List ctxIdsToDelete = cfIdsOfOldProfile.stream().map(cfId -> JacksonUtil.writeValueAsString(new CalculatedFieldCtxId(cfId.getId(), entityId.getId()))).toList(); - rocksDBService.deleteAll(ctxIdsToDelete); - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> { CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(cfId.getId(), entityId.getId()); @@ -269,7 +254,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, newProfileId) .stream() .map(cfId -> calculatedFields.computeIfAbsent(cfId, id -> calculatedFieldService.findById(tenantId, id))) - .forEach(cf -> initializeStateForEntity(tenantId, cf, entityId, callback)); + .forEach(cf -> initializeStateForEntity(cf, entityId, callback)); } catch (Exception e) { log.trace("Failed to process entity type update msg: [{}]", proto, e); } @@ -328,11 +313,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); } - private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { + private void initializeStateForEntity(CalculatedField calculatedField, EntityId entityId, TbCallback callback) { Map arguments = calculatedField.getConfiguration().getArguments(); Map argumentValues = new HashMap<>(); AtomicInteger remaining = new AtomicInteger(arguments.size()); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedField, argument), new FutureCallback<>() { + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedField, entityId, argument), new FutureCallback<>() { @Override public void onSuccess(ArgumentEntry result) { argumentValues.put(key, result); @@ -349,12 +334,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor)); } - private ListenableFuture fetchArgumentValue(CalculatedField calculatedField, Argument argument) { + private ListenableFuture fetchArgumentValue(CalculatedField calculatedField, EntityId targetEntityId, Argument argument) { TenantId tenantId = calculatedField.getTenantId(); - EntityId cfEntityId = calculatedField.getEntityId(); EntityId argumentEntityId = argument.getEntityId(); EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) - ? cfEntityId + ? targetEntityId : argumentEntityId; if (CalculatedFieldType.LAST_RECORDS.equals(calculatedField.getType())) { return fetchLastRecords(tenantId, entityId, argument); @@ -371,11 +355,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, endTs, 0, limit, Aggregation.NONE); ListenableFuture> lastRecordsFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); - return Futures.transform(lastRecordsFuture, lastRecords -> { - ArgumentEntry argumentEntry = new ArgumentEntry(); - argumentEntry.setKvEntries(lastRecords); - return argumentEntry; - }, calculatedFieldExecutor); + return Futures.transform(lastRecordsFuture, ArgumentEntry::createArgumentEntry, calculatedFieldExecutor); } private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { @@ -394,13 +374,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldExecutor); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; - return Futures.transform(kvEntryFuture, kvEntry -> { - ArgumentEntry argumentEntry = new ArgumentEntry(); - if (kvEntry.isPresent()) { - argumentEntry.setKvEntry(kvEntry.orElse(null)); - } - return argumentEntry; - }, calculatedFieldExecutor); + return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createArgumentEntry(kvEntry.orElse(null)), calculatedFieldExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { @@ -429,12 +403,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.put(ctxId, calculatedFieldCtx); rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); - CalculationContext ctx = CalculationContext.builder() - .tenantId(calculatedField.getTenantId()) - .configuration(calculatedField.getConfiguration()) - .tbelInvokeService(tbelInvokeService) - .build(); - ListenableFuture resultFuture = state.performCalculation(ctx); + ListenableFuture resultFuture = state.performCalculation(buildCalculationContext(calculatedField)); Futures.addCallback(resultFuture, new FutureCallback<>() { @Override public void onSuccess(CalculatedFieldResult result) { @@ -464,6 +433,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private CalculationContext buildCalculationContext(CalculatedField calculatedField) { + CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); + return CalculationContext.builder() + .tenantId(calculatedField.getTenantId()) + .arguments(configuration.getArguments()) + .output(configuration.getOutput()) + .expression(configuration.getExpression()) + .tbelInvokeService(tbelInvokeService) + .build(); + } + private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { ObjectNode payload = JacksonUtil.newObjectNode(); Map resultMap = calculatedFieldResult.getResultMap(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 29e3417bde..3097056d11 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -15,16 +15,25 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import lombok.Data; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.List; -@Data -public class ArgumentEntry { +public interface ArgumentEntry { - private KvEntry kvEntry; - private List kvEntries; + Object getValue(); + + static ArgumentEntry createArgumentEntry(KvEntry kvEntry) { + if (kvEntry instanceof TsKvEntry tsKvEntry) { + return new LastRecordsArgumentEntry(List.of(tsKvEntry)); + } else { + return new KvArgumentEntry(kvEntry); + } + } + + static ArgumentEntry createArgumentEntry(List kvEntries) { + return new LastRecordsArgumentEntry(kvEntries); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java new file mode 100644 index 0000000000..bac318a1b9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.service.cf.CalculatedFieldResult; + +import java.util.HashMap; +import java.util.Map; + +public abstract class BaseCalculatedFieldState implements CalculatedFieldState { + + protected Map arguments; + + public BaseCalculatedFieldState() { + } + + @Override + public void initState(Map argumentValues) { + if (arguments == null) { + arguments = new HashMap<>(); + } +// argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); + } + + protected CalculatedFieldResult buildResult(Output output, Map resultMap) { + CalculatedFieldResult result = new CalculatedFieldResult(); + result.setType(output.getType()); + result.setScope(output.getScope()); + result.setResultMap(resultMap); + return result; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java index 656763ea48..aaabaec0d4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java @@ -18,7 +18,9 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.Builder; import lombok.Data; import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; @@ -29,7 +31,9 @@ import java.util.Map; public class CalculationContext { private TenantId tenantId; - private CalculatedFieldConfiguration configuration; + private Map arguments; + private Output output; + private String expression; private TbelInvokeService tbelInvokeService; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java new file mode 100644 index 0000000000..0bd21d452e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.Data; +import org.thingsboard.server.common.data.kv.KvEntry; + +@Data +public class KvArgumentEntry implements ArgumentEntry { + + private final KvEntry kvEntry; + + @Override + public Object getValue() { + return kvEntry; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java new file mode 100644 index 0000000000..8729f022fa --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.Data; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +@Data +public class LastRecordsArgumentEntry implements ArgumentEntry { + + private final List kvEntries; + + @Override + public Object getValue() { + return kvEntries; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java index 2f26d71f91..0428d0823d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java @@ -15,9 +15,11 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import aj.org.objectweb.asm.TypeReference; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; @@ -33,7 +35,7 @@ import java.util.Map; import java.util.stream.Collectors; @Data -public class LastRecordsCalculatedFieldState implements CalculatedFieldState { +public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { private Map> arguments; @@ -53,21 +55,16 @@ public class LastRecordsCalculatedFieldState implements CalculatedFieldState { } argumentValues.forEach((key, argumentEntry) -> { List tsKvEntryList = arguments.computeIfAbsent(key, k -> new ArrayList<>()); - tsKvEntryList.addAll(argumentEntry.getKvEntries()); +// tsKvEntryList.addAll(argumentEntry.getKvEntries()); }); } @Override public ListenableFuture performCalculation(CalculationContext ctx) { - CalculatedFieldConfiguration configuration = ctx.getConfiguration(); - Map configArguments = configuration.getArguments(); - Output output = configuration.getOutput(); - Map resultMap = new HashMap<>(); - arguments.replaceAll((key, entries) -> { - int limit = configArguments.get(key).getLimit(); + int limit = ctx.getArguments().get(key).getLimit(); List limitedEntries = entries.stream() .sorted(Comparator.comparingLong(TsKvEntry::getTs).reversed()) .limit(limit) @@ -79,13 +76,7 @@ public class LastRecordsCalculatedFieldState implements CalculatedFieldState { return limitedEntries; }); - - CalculatedFieldResult calculatedFieldResult = new CalculatedFieldResult(); - calculatedFieldResult.setType(output.getType()); - calculatedFieldResult.setScope(output.getScope()); - calculatedFieldResult.setResultMap(resultMap); - - return Futures.immediateFuture(calculatedFieldResult); + return Futures.immediateFuture(buildResult(ctx.getOutput(), resultMap)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 0e9b00ad7d..047bfd0f8c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -35,16 +35,11 @@ import java.util.Map; @Data @Slf4j -public class ScriptCalculatedFieldState implements CalculatedFieldState { +public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { @JsonIgnore private CalculatedFieldScriptEngine calculatedFieldScriptEngine; - private Map arguments; - - public ScriptCalculatedFieldState() { - } - @Override public CalculatedFieldType getType() { return CalculatedFieldType.SCRIPT; @@ -52,50 +47,35 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { @Override - public void initState(Map argumentValues) { - if (arguments == null) { - arguments = new HashMap<>(); - } - argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); - } + public ListenableFuture performCalculation(CalculationContext ctx) { + if (isValid(this.arguments, ctx.getArguments())) { + if (calculatedFieldScriptEngine == null) { + initEngine(ctx.getTenantId(), ctx.getExpression(), ctx.getTbelInvokeService()); + } + ListenableFuture resultFuture = calculatedFieldScriptEngine.executeScriptAsync(this.arguments); - @Override - public ListenableFuture performCalculation(CalculationContext ctx) { - CalculatedFieldConfiguration calculatedFieldConfiguration = ctx.getConfiguration(); - TbelInvokeService tbelInvokeService = ctx.getTbelInvokeService(); + return Futures.transform(resultFuture, result -> { + Map resultMap = result instanceof Map + ? JacksonUtil.convertValue(result, Map.class) + : new HashMap<>(); - if (tbelInvokeService == null) { - throw new IllegalArgumentException("TBEL script engine is disabled!"); + return buildResult(ctx.getOutput(), resultMap); + }, MoreExecutors.directExecutor()); } + return null; + } - if (calculatedFieldScriptEngine == null) { - initEngine(ctx.getTenantId(), calculatedFieldConfiguration, tbelInvokeService); + private void initEngine(TenantId tenantId, String expression, TbelInvokeService tbelInvokeService) { + if (tbelInvokeService == null) { + throw new IllegalArgumentException("TBEL script engine is disabled!"); } - ListenableFuture resultFuture = calculatedFieldScriptEngine.executeScriptAsync(arguments); - - return Futures.transform(resultFuture, result -> { - Output output = calculatedFieldConfiguration.getOutput(); - Map resultMap = result instanceof Map - ? JacksonUtil.convertValue(result, Map.class) - : new HashMap<>(); - - CalculatedFieldResult calculatedFieldResult = new CalculatedFieldResult(); - calculatedFieldResult.setType(output.getType()); - calculatedFieldResult.setScope(output.getScope()); - calculatedFieldResult.setResultMap(resultMap); - - return calculatedFieldResult; - }, MoreExecutors.directExecutor()); - } - - private void initEngine(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( tenantId, tbelInvokeService, - calculatedFieldConfiguration.getExpression(), - arguments.keySet().toArray(new String[0]) + expression, + this.arguments.keySet().toArray(new String[0]) ); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index d3d3b5f636..fbdd1eb354 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -31,36 +31,17 @@ import java.util.HashMap; import java.util.Map; @Data -public class SimpleCalculatedFieldState implements CalculatedFieldState { - - private Map arguments; - - public SimpleCalculatedFieldState() { - } +public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { @Override public CalculatedFieldType getType() { return CalculatedFieldType.SIMPLE; } - @Override - public void initState(Map argumentValues) { - if (arguments == null) { - arguments = new HashMap<>(); - } - argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); - } - @Override public ListenableFuture performCalculation(CalculationContext ctx) { - CalculatedFieldConfiguration calculatedFieldConfiguration = ctx.getConfiguration(); - - Output output = calculatedFieldConfiguration.getOutput(); - Map arguments = calculatedFieldConfiguration.getArguments(); - - if (isValid(this.arguments, arguments)) { - CalculatedFieldResult result = new CalculatedFieldResult(); - String expression = calculatedFieldConfiguration.getExpression(); + if (isValid(this.arguments, ctx.getArguments())) { + String expression = ctx.getExpression(); ThreadLocal customExpression = new ThreadLocal<>(); var expr = customExpression.get(); if (expr == null) { @@ -76,10 +57,8 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { double expressionResult = expr.evaluate(); - result.setType(output.getType()); - result.setScope(output.getScope()); - result.setResultMap(Map.of(output.getName(), expressionResult)); - return Futures.immediateFuture(result); + Output output = ctx.getOutput(); + return Futures.immediateFuture(buildResult(output, Map.of(output.getName(), expressionResult))); } return null; } From d684c8777a2d7ec438e8a4b7b7de0a6841eaae49 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 26 Nov 2024 17:33:01 +0200 Subject: [PATCH 09/21] improved usage of calculation ctx --- .../cf/CalculatedFieldExecutionService.java | 2 +- .../service/cf/CalculatedFieldResult.java | 8 +- ...efaultCalculatedFieldExecutionService.java | 95 +++++++++---------- ...Ctx.java => CalculatedFieldEntityCtx.java} | 8 +- ...d.java => CalculatedFieldEntityCtxId.java} | 2 +- .../service/cf/ctx/state/ArgumentEntry.java | 26 +++-- ...KvArgumentEntry.java => ArgumentType.java} | 15 +-- .../ctx/state/BaseCalculatedFieldState.java | 30 +++--- .../cf/ctx/state/CalculatedFieldCtx.java | 72 ++++++++++++++ .../state/CalculatedFieldScriptEngine.java | 7 +- .../cf/ctx/state/CalculatedFieldState.java | 8 +- .../CalculatedFieldTbelScriptEngine.java | 18 +++- .../ctx/state/LastRecordsArgumentEntry.java | 16 +++- .../LastRecordsCalculatedFieldState.java | 46 +++++---- .../ctx/state/ScriptCalculatedFieldState.java | 47 ++------- .../ctx/state/SimpleCalculatedFieldState.java | 9 +- ...ext.java => SingleValueArgumentEntry.java} | 32 +++---- .../queue/DefaultTbCoreConsumerService.java | 2 +- .../data/cf/configuration/Argument.java | 1 - .../BaseCalculatedFieldConfiguration.java | 4 - 20 files changed, 257 insertions(+), 191 deletions(-) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/{CalculatedFieldCtx.java => CalculatedFieldEntityCtx.java} (79%) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/{CalculatedFieldCtxId.java => CalculatedFieldEntityCtxId.java} (90%) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/{KvArgumentEntry.java => ArgumentType.java} (72%) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/{CalculationContext.java => SingleValueArgumentEntry.java} (51%) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 6b7b12655f..6d7cf0e741 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -29,6 +29,6 @@ public interface CalculatedFieldExecutionService { void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); - void onEntityTypeChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); + void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index 87f1d08a84..1f8a06c8fa 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -21,13 +21,15 @@ import org.thingsboard.server.common.data.AttributeScope; import java.util.Map; @Data -public class CalculatedFieldResult { +public final class CalculatedFieldResult { private String type; private AttributeScope scope; private Map resultMap; - public CalculatedFieldResult() { + public CalculatedFieldResult(String type, AttributeScope scope, Map resultMap) { + this.type = type; + this.scope = scope; + this.resultMap = resultMap == null ? Map.of() : Map.copyOf(resultMap); } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 322443c848..0d269f032f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -71,11 +71,11 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtx; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtxId; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; -import org.thingsboard.server.service.cf.ctx.state.CalculationContext; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.LastRecordsCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; @@ -115,7 +115,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); - private final ConcurrentMap states = new ConcurrentHashMap<>(); + private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); + private final ConcurrentMap states = new ConcurrentHashMap<>(); private static final int MAX_LAST_RECORDS_VALUE = 1024; @@ -188,22 +189,24 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId entityId = cf.getEntityId(); calculatedFields.put(calculatedFieldId, cf); calculatedFieldLinks.put(calculatedFieldId, links); + CalculatedFieldCtx calculatedFieldCtx = new CalculatedFieldCtx(cf, tbelInvokeService); + calculatedFieldsCtx.put(calculatedFieldId, calculatedFieldCtx); switch (entityId.getEntityType()) { case ASSET, DEVICE -> { log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); - initializeStateForEntity(cf, entityId, callback); + initializeStateForEntity(calculatedFieldCtx, entityId, callback); } case ASSET_PROFILE -> { log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - assetIds.forEach(assetId -> initializeStateForEntity(cf, assetId, callback)); + assetIds.forEach(assetId -> initializeStateForEntity(calculatedFieldCtx, assetId, callback)); } case DEVICE_PROFILE -> { log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - deviceIds.forEach(deviceId -> initializeStateForEntity(cf, deviceId, callback)); + deviceIds.forEach(deviceId -> initializeStateForEntity(calculatedFieldCtx, deviceId, callback)); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -224,10 +227,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { - CalculatedField calculatedField = calculatedFields.computeIfAbsent(calculatedFieldId, id -> calculatedFieldService.findById(tenantId, id)); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> { + CalculatedField calculatedField = calculatedFields.computeIfAbsent(id, cfId -> calculatedFieldService.findById(tenantId, id)); + return new CalculatedFieldCtx(calculatedField, tbelInvokeService); + }); Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createArgumentEntry(entry.getValue()))); - updateOrInitializeState(calculatedField, calculatedField.getEntityId(), argumentValues); + .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); + updateOrInitializeState(calculatedFieldCtx, calculatedFieldCtx.getEntityId(), argumentValues); log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); @@ -235,7 +241,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onEntityTypeChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { + public void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); @@ -246,15 +252,15 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> { - CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(cfId.getId(), entityId.getId()); + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); states.remove(ctxId); rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); }); calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, newProfileId) .stream() - .map(cfId -> calculatedFields.computeIfAbsent(cfId, id -> calculatedFieldService.findById(tenantId, id))) - .forEach(cf -> initializeStateForEntity(cf, entityId, callback)); + .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) + .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } catch (Exception e) { log.trace("Failed to process entity type update msg: [{}]", proto, e); } @@ -267,6 +273,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas onCalculatedFieldDelete(newCalculatedField.getId(), callback); } else { calculatedFields.put(newCalculatedField.getId(), newCalculatedField); + calculatedFieldsCtx.put(newCalculatedField.getId(), new CalculatedFieldCtx(newCalculatedField, tbelInvokeService)); callback.onSuccess(); shouldReinit = false; } @@ -277,6 +284,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas try { calculatedFieldLinks.remove(calculatedFieldId); calculatedFields.remove(calculatedFieldId); + calculatedFieldsCtx.remove(calculatedFieldId); states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); List statesToRemove = states.keySet().stream() .filter(ctxId -> !calculatedFields.containsKey(new CalculatedFieldId(ctxId.cfId()))) @@ -309,20 +317,20 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldCtx.class))); + rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class))); states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); } - private void initializeStateForEntity(CalculatedField calculatedField, EntityId entityId, TbCallback callback) { - Map arguments = calculatedField.getConfiguration().getArguments(); + private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { + Map arguments = calculatedFieldCtx.getArguments(); Map argumentValues = new HashMap<>(); AtomicInteger remaining = new AtomicInteger(arguments.size()); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedField, entityId, argument), new FutureCallback<>() { + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedFieldCtx, entityId, argument), new FutureCallback<>() { @Override public void onSuccess(ArgumentEntry result) { argumentValues.put(key, result); if (remaining.decrementAndGet() == 0) { - updateOrInitializeState(calculatedField, entityId, argumentValues); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); } } @@ -334,28 +342,28 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor)); } - private ListenableFuture fetchArgumentValue(CalculatedField calculatedField, EntityId targetEntityId, Argument argument) { - TenantId tenantId = calculatedField.getTenantId(); + private ListenableFuture fetchArgumentValue(CalculatedFieldCtx calculatedFieldCtx, EntityId targetEntityId, Argument argument) { + TenantId tenantId = calculatedFieldCtx.getTenantId(); EntityId argumentEntityId = argument.getEntityId(); EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; - if (CalculatedFieldType.LAST_RECORDS.equals(calculatedField.getType())) { + if (CalculatedFieldType.LAST_RECORDS.equals(calculatedFieldCtx.getCfType())) { return fetchLastRecords(tenantId, entityId, argument); } return fetchKvEntry(tenantId, entityId, argument); } private ListenableFuture fetchLastRecords(TenantId tenantId, EntityId entityId, Argument argument) { - long startTs = Math.max(argument.getStartTs(), 0); + long currentTime = System.currentTimeMillis(); long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); - long endTs = startTs + timeWindow; + long startTs = currentTime - timeWindow; int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); - ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, endTs, 0, limit, Aggregation.NONE); + ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); ListenableFuture> lastRecordsFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); - return Futures.transform(lastRecordsFuture, ArgumentEntry::createArgumentEntry, calculatedFieldExecutor); + return Futures.transform(lastRecordsFuture, ArgumentEntry::createLastRecordsArgument, calculatedFieldExecutor); } private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { @@ -374,7 +382,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldExecutor); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; - return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createArgumentEntry(kvEntry.orElse(null)), calculatedFieldExecutor); + return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { @@ -389,32 +397,32 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return new StringDataEntry(key, defaultValue); } - private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { - CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(calculatedField.getUuidId(), entityId.getId()); - CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(ctxId, null)); + private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues) { + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(calculatedFieldCtx.getCfId().getId(), entityId.getId()); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctx -> new CalculatedFieldEntityCtx(entityCtxId, null)); - CalculatedFieldState state = calculatedFieldCtx.getState(); + CalculatedFieldState state = calculatedFieldEntityCtx.getState(); if (state == null) { - state = createStateByType(calculatedField.getType()); + state = createStateByType(calculatedFieldCtx.getCfType()); } state.initState(argumentValues); - calculatedFieldCtx.setState(state); - states.put(ctxId, calculatedFieldCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); + calculatedFieldEntityCtx.setState(state); + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - ListenableFuture resultFuture = state.performCalculation(buildCalculationContext(calculatedField)); + ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); Futures.addCallback(resultFuture, new FutureCallback<>() { @Override public void onSuccess(CalculatedFieldResult result) { if (result != null) { - pushMsgToRuleEngine(calculatedField.getTenantId(), entityId, result); + pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); } } @Override public void onFailure(Throwable t) { - log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedField.getId(), entityId, t); + log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); } }, MoreExecutors.directExecutor()); @@ -433,17 +441,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private CalculationContext buildCalculationContext(CalculatedField calculatedField) { - CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); - return CalculationContext.builder() - .tenantId(calculatedField.getTenantId()) - .arguments(configuration.getArguments()) - .output(configuration.getOutput()) - .expression(configuration.getExpression()) - .tbelInvokeService(tbelInvokeService) - .build(); - } - private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { ObjectNode payload = JacksonUtil.newObjectNode(); Map resultMap = calculatedFieldResult.getResultMap(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java similarity index 79% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java index 4b2a6c918f..7a8384b6bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java @@ -19,15 +19,15 @@ import lombok.Data; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @Data -public class CalculatedFieldCtx { +public class CalculatedFieldEntityCtx { - private CalculatedFieldCtxId id; + private CalculatedFieldEntityCtxId id; private CalculatedFieldState state; - public CalculatedFieldCtx() { + public CalculatedFieldEntityCtx() { } - public CalculatedFieldCtx(CalculatedFieldCtxId id, CalculatedFieldState state) { + public CalculatedFieldEntityCtx(CalculatedFieldEntityCtxId id, CalculatedFieldState state) { this.id = id; this.state = state; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java similarity index 90% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java index a316c54b76..f7c451efee 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java @@ -17,5 +17,5 @@ package org.thingsboard.server.service.cf.ctx; import java.util.UUID; -public record CalculatedFieldCtxId(UUID cfId, UUID entityId) { +public record CalculatedFieldEntityCtxId(UUID cfId, UUID entityId) { } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 3097056d11..6218f38edf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -15,25 +15,35 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.List; +import java.util.stream.Collectors; +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = SingleValueArgumentEntry.class, name = "SINGLE_VALUE"), + @JsonSubTypes.Type(value = LastRecordsArgumentEntry.class, name = "LAST_RECORDS") +}) public interface ArgumentEntry { + ArgumentType getType(); + Object getValue(); - static ArgumentEntry createArgumentEntry(KvEntry kvEntry) { - if (kvEntry instanceof TsKvEntry tsKvEntry) { - return new LastRecordsArgumentEntry(List.of(tsKvEntry)); - } else { - return new KvArgumentEntry(kvEntry); - } + static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { + return new SingleValueArgumentEntry(kvEntry.getValue()); } - static ArgumentEntry createArgumentEntry(List kvEntries) { - return new LastRecordsArgumentEntry(kvEntries); + static ArgumentEntry createLastRecordsArgument(List kvEntries) { + return new LastRecordsArgumentEntry(kvEntries.stream() .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue))); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java similarity index 72% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java index 0bd21d452e..f2f0eac60d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java @@ -15,17 +15,6 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import lombok.Data; -import org.thingsboard.server.common.data.kv.KvEntry; - -@Data -public class KvArgumentEntry implements ArgumentEntry { - - private final KvEntry kvEntry; - - @Override - public Object getValue() { - return kvEntry; - } - +public enum ArgumentType { + SINGLE_VALUE, LAST_RECORDS } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index bac318a1b9..54df89e757 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -15,34 +15,36 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.service.cf.CalculatedFieldResult; - import java.util.HashMap; import java.util.Map; +import java.util.stream.Collectors; public abstract class BaseCalculatedFieldState implements CalculatedFieldState { - protected Map arguments; + protected Map arguments; public BaseCalculatedFieldState() { } + @Override + public Map getArguments() { + return this.arguments; + } + @Override public void initState(Map argumentValues) { if (arguments == null) { arguments = new HashMap<>(); } -// argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); - } - - protected CalculatedFieldResult buildResult(Output output, Map resultMap) { - CalculatedFieldResult result = new CalculatedFieldResult(); - result.setType(output.getType()); - result.setScope(output.getScope()); - result.setResultMap(resultMap); - return result; + arguments.putAll( + argumentValues.entrySet().stream() + .peek(entry -> { + if (entry.getValue() instanceof LastRecordsArgumentEntry) { + throw new IllegalArgumentException("Last records argument entry is not allowed for single calculated field state"); + } + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) + ); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java new file mode 100644 index 0000000000..10395a0d5d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -0,0 +1,72 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.Data; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Map; + +@Data +public class CalculatedFieldCtx { + + private CalculatedFieldId cfId; + private TenantId tenantId; + private EntityId entityId; + private CalculatedFieldType cfType; + private Map arguments; + private Output output; + private String expression; + private TbelInvokeService tbelInvokeService; + private CalculatedFieldScriptEngine calculatedFieldScriptEngine; + + public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService) { + this.cfId = calculatedField.getId(); + this.tenantId = calculatedField.getTenantId(); + this.entityId = calculatedField.getEntityId(); + this.cfType = calculatedField.getType(); + CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); + this.arguments = configuration.getArguments(); + this.output = configuration.getOutput(); + this.expression = configuration.getExpression(); + this.tbelInvokeService = tbelInvokeService; + if (CalculatedFieldType.SCRIPT.equals(calculatedField.getType())) { + this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); + } + } + + private CalculatedFieldScriptEngine initEngine(TenantId tenantId, String expression, TbelInvokeService tbelInvokeService) { + if (tbelInvokeService == null) { + throw new IllegalArgumentException("TBEL script engine is disabled!"); + } + + return new CalculatedFieldTbelScriptEngine( + tenantId, + tbelInvokeService, + expression, + arguments.keySet().toArray(new String[0]) + ); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java index 6b54536019..212d971398 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java @@ -16,13 +16,16 @@ package org.thingsboard.server.service.cf.ctx.state; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.kv.KvEntry; import java.util.Map; public interface CalculatedFieldScriptEngine { - ListenableFuture executeScriptAsync(Map arguments); + ListenableFuture executeScriptAsync(Map arguments); + + ListenableFuture> executeToMapAsync(Map arguments); + + ListenableFuture> executeToMapTransform(Object result); void destroy(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index e4c4440921..f0882cbea4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -41,12 +41,14 @@ public interface CalculatedFieldState { @JsonIgnore CalculatedFieldType getType(); - default boolean isValid(Map argumentValues, Map arguments) { - return argumentValues.keySet().containsAll(arguments.keySet()); + Map getArguments(); + + default boolean isValid(Map arguments) { + return getArguments().keySet().containsAll(arguments.keySet()); } void initState(Map argumentValues); - ListenableFuture performCalculation(CalculationContext ctx); + ListenableFuture performCalculation(CalculatedFieldCtx ctx); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java index 5cc58b2a95..363eefd44e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.ScriptType; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.id.TenantId; @@ -52,9 +53,9 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng } @Override - public ListenableFuture executeScriptAsync(Map arguments) { + public ListenableFuture executeScriptAsync(Map arguments) { log.trace("execute script async, arguments {}", arguments); - Object[] args = arguments.values().stream().map(KvEntry::getValue).toArray(); + Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); return Futures.transformAsync(tbelInvokeService.invokeScript(tenantId, null, this.scriptId, args), o -> { try { @@ -71,6 +72,19 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng }, MoreExecutors.directExecutor()); } + @Override + public ListenableFuture> executeToMapAsync(Map arguments) { + return Futures.transformAsync(executeScriptAsync(arguments), this::executeToMapTransform, MoreExecutors.directExecutor()); + } + + @Override + public ListenableFuture> executeToMapTransform(Object result) { + if (result instanceof Map) { + return Futures.immediateFuture((Map) result); + } + throw new IllegalArgumentException("Wrong result type: [" + result.getClass().getName() + "]"); + } + @Override public void destroy() { tbelInvokeService.release(this.scriptId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java index 8729f022fa..39da1838bc 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java @@ -15,19 +15,27 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import lombok.AllArgsConstructor; import lombok.Data; -import org.thingsboard.server.common.data.kv.TsKvEntry; +import lombok.NoArgsConstructor; -import java.util.List; +import java.util.Map; @Data +@NoArgsConstructor +@AllArgsConstructor public class LastRecordsArgumentEntry implements ArgumentEntry { - private final List kvEntries; + private Map tsRecords; + + @Override + public ArgumentType getType() { + return ArgumentType.LAST_RECORDS; + } @Override public Object getValue() { - return kvEntries; + return tsRecords.values(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java index 0428d0823d..437707bfdb 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java @@ -15,14 +15,10 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import aj.org.objectweb.asm.TypeReference; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -37,8 +33,6 @@ import java.util.stream.Collectors; @Data public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { - private Map> arguments; - public LastRecordsCalculatedFieldState() { } @@ -47,36 +41,46 @@ public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { return CalculatedFieldType.LAST_RECORDS; } - @Override public void initState(Map argumentValues) { if (arguments == null) { arguments = new HashMap<>(); } argumentValues.forEach((key, argumentEntry) -> { - List tsKvEntryList = arguments.computeIfAbsent(key, k -> new ArrayList<>()); -// tsKvEntryList.addAll(argumentEntry.getKvEntries()); + LastRecordsArgumentEntry existingArgumentEntry = (LastRecordsArgumentEntry) + arguments.computeIfAbsent(key, k -> new LastRecordsArgumentEntry(new HashMap<>())); + if (argumentEntry instanceof LastRecordsArgumentEntry lastRecordsArgumentEntry) { + existingArgumentEntry.getTsRecords().putAll(lastRecordsArgumentEntry.getTsRecords()); + } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry + && singleValueArgumentEntry.getValue() instanceof TsKvEntry tsKvEntry) { + existingArgumentEntry.getTsRecords().put(tsKvEntry.getTs(), tsKvEntry.getValue()); + } }); } - @Override - public ListenableFuture performCalculation(CalculationContext ctx) { + public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { Map resultMap = new HashMap<>(); - arguments.replaceAll((key, entries) -> { + arguments.replaceAll((key, argumentEntry) -> { int limit = ctx.getArguments().get(key).getLimit(); - List limitedEntries = entries.stream() - .sorted(Comparator.comparingLong(TsKvEntry::getTs).reversed()) - .limit(limit) - .collect(Collectors.toList()); - Map valueWithTs = limitedEntries.stream() - .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue)); - resultMap.put(key, valueWithTs); + // TODO: implement removing if size > limit + + +// List limitedEntries = entries.stream() +// .sorted(Comparator.comparingLong(TsKvEntry::getTs).reversed()) +// .limit(limit) +// .collect(Collectors.toList()); +// +// Map valueWithTs = limitedEntries.stream() +// .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue)); +// resultMap.put(key, valueWithTs); - return limitedEntries; +// return new LastRecordsArgumentEntry(limitedEntries); + return null; }); - return Futures.immediateFuture(buildResult(ctx.getOutput(), resultMap)); + Output output = ctx.getOutput(); + return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), resultMap)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 047bfd0f8c..16905930d3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -15,68 +15,37 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; -import java.util.HashMap; import java.util.Map; @Data @Slf4j public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { - @JsonIgnore - private CalculatedFieldScriptEngine calculatedFieldScriptEngine; - @Override public CalculatedFieldType getType() { return CalculatedFieldType.SCRIPT; } - @Override - public ListenableFuture performCalculation(CalculationContext ctx) { - if (isValid(this.arguments, ctx.getArguments())) { - if (calculatedFieldScriptEngine == null) { - initEngine(ctx.getTenantId(), ctx.getExpression(), ctx.getTbelInvokeService()); - } - - ListenableFuture resultFuture = calculatedFieldScriptEngine.executeScriptAsync(this.arguments); - - return Futures.transform(resultFuture, result -> { - Map resultMap = result instanceof Map - ? JacksonUtil.convertValue(result, Map.class) - : new HashMap<>(); - - return buildResult(ctx.getOutput(), resultMap); - }, MoreExecutors.directExecutor()); + public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { + Output output = ctx.getOutput(); + if (isValid(ctx.getArguments())) { + ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(this.arguments); + return Futures.transform(resultFuture, + result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), + MoreExecutors.directExecutor() + ); } return null; } - private void initEngine(TenantId tenantId, String expression, TbelInvokeService tbelInvokeService) { - if (tbelInvokeService == null) { - throw new IllegalArgumentException("TBEL script engine is disabled!"); - } - - calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( - tenantId, - tbelInvokeService, - expression, - this.arguments.keySet().toArray(new String[0]) - ); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index fbdd1eb354..64eb409d3a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -22,7 +22,6 @@ import net.objecthunter.exp4j.Expression; import net.objecthunter.exp4j.ExpressionBuilder; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -39,8 +38,8 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { } @Override - public ListenableFuture performCalculation(CalculationContext ctx) { - if (isValid(this.arguments, ctx.getArguments())) { + public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { + if (isValid(ctx.getArguments())) { String expression = ctx.getExpression(); ThreadLocal customExpression = new ThreadLocal<>(); var expr = customExpression.get(); @@ -52,13 +51,13 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { customExpression.set(expr); } Map variables = new HashMap<>(); - this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValueAsString()))); + this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(((KvEntry) v.getValue()).getValueAsString()))); expr.setVariables(variables); double expressionResult = expr.evaluate(); Output output = ctx.getOutput(); - return Futures.immediateFuture(buildResult(output, Map.of(output.getName(), expressionResult))); + return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), Map.of(output.getName(), expressionResult))); } return null; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java similarity index 51% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index aaabaec0d4..504d213748 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -15,25 +15,25 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import lombok.Builder; +import lombok.AllArgsConstructor; import lombok.Data; -import org.thingsboard.script.api.tbel.TbelInvokeService; -import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; - -import java.util.Map; +import lombok.NoArgsConstructor; @Data -@Builder -public class CalculationContext { +@NoArgsConstructor +@AllArgsConstructor +public class SingleValueArgumentEntry implements ArgumentEntry { + + private Object value; + + @Override + public ArgumentType getType() { + return ArgumentType.SINGLE_VALUE; + } - private TenantId tenantId; - private Map arguments; - private Output output; - private String expression; - private TbelInvokeService tbelInvokeService; + @Override + public Object getValue() { + return value; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 3caaab6613..392e2637d1 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -685,7 +685,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityTypeChanged(profileUpdateMsg, callback)); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChanged(profileUpdateMsg, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index 0d70591a38..f34f5e9cb7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -29,7 +29,6 @@ public class Argument { private String defaultValue; private int limit; - private long startTs; private long timeWindow; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index f7cc53b7cf..f311fb737b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -102,7 +102,6 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argumentNode.put("scope", String.valueOf(argument.getScope())); argumentNode.put("defaultValue", argument.getDefaultValue()); argumentNode.put("limit", String.valueOf(argument.getLimit())); - argumentNode.put("startTs", String.valueOf(argument.getStartTs())); argumentNode.put("timeWindow", String.valueOf(argument.getTimeWindow())); }); @@ -153,9 +152,6 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel if (argumentNode.hasNonNull("limit")) { argument.setLimit(argumentNode.get("limit").asInt()); } - if (argumentNode.hasNonNull("startTs")) { - argument.setStartTs(argumentNode.get("startTs").asLong()); - } if (argumentNode.hasNonNull("timeWindow")) { argument.setTimeWindow(argumentNode.get("timeWindow").asInt()); } From f9db64a14d036cbd6a5ffdbe3a733b177e9d081b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 27 Nov 2024 12:10:04 +0200 Subject: [PATCH 10/21] added deletion from map if ts out of timewindow --- .../service/cf/ctx/state/ArgumentEntry.java | 8 +++- .../ctx/state/LastRecordsArgumentEntry.java | 6 ++- .../LastRecordsCalculatedFieldState.java | 46 ++++++++----------- .../ctx/state/SimpleCalculatedFieldState.java | 4 +- .../ctx/state/SingleValueArgumentEntry.java | 20 ++++++-- 5 files changed, 46 insertions(+), 38 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 6218f38edf..6a62c0b1e3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -15,12 +15,14 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.List; +import java.util.TreeMap; import java.util.stream.Collectors; @JsonTypeInfo( @@ -34,16 +36,18 @@ import java.util.stream.Collectors; }) public interface ArgumentEntry { + @JsonIgnore ArgumentType getType(); Object getValue(); static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { - return new SingleValueArgumentEntry(kvEntry.getValue()); + return new SingleValueArgumentEntry(kvEntry); } static ArgumentEntry createLastRecordsArgument(List kvEntries) { - return new LastRecordsArgumentEntry(kvEntries.stream() .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue))); + return new LastRecordsArgumentEntry(kvEntries.stream(). + collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue, (oldValue, newValue) -> newValue, TreeMap::new))); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java index 39da1838bc..93fabd3cb8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java @@ -15,24 +15,26 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import java.util.Map; +import java.util.TreeMap; @Data @NoArgsConstructor @AllArgsConstructor public class LastRecordsArgumentEntry implements ArgumentEntry { - private Map tsRecords; + private TreeMap tsRecords; @Override public ArgumentType getType() { return ArgumentType.LAST_RECORDS; } + @JsonIgnore @Override public Object getValue() { return tsRecords.values(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java index 437707bfdb..dd69790236 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java @@ -19,16 +19,13 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; -import java.util.ArrayList; -import java.util.Comparator; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.stream.Collectors; +import java.util.TreeMap; @Data public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { @@ -44,16 +41,15 @@ public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { @Override public void initState(Map argumentValues) { if (arguments == null) { - arguments = new HashMap<>(); + arguments = new TreeMap<>(); } argumentValues.forEach((key, argumentEntry) -> { LastRecordsArgumentEntry existingArgumentEntry = (LastRecordsArgumentEntry) - arguments.computeIfAbsent(key, k -> new LastRecordsArgumentEntry(new HashMap<>())); + arguments.computeIfAbsent(key, k -> new LastRecordsArgumentEntry(new TreeMap<>())); if (argumentEntry instanceof LastRecordsArgumentEntry lastRecordsArgumentEntry) { existingArgumentEntry.getTsRecords().putAll(lastRecordsArgumentEntry.getTsRecords()); - } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry - && singleValueArgumentEntry.getValue() instanceof TsKvEntry tsKvEntry) { - existingArgumentEntry.getTsRecords().put(tsKvEntry.getTs(), tsKvEntry.getValue()); + } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + existingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); } }); } @@ -61,26 +57,22 @@ public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { Map resultMap = new HashMap<>(); - arguments.replaceAll((key, argumentEntry) -> { - int limit = ctx.getArguments().get(key).getLimit(); - - // TODO: implement removing if size > limit - - -// List limitedEntries = entries.stream() -// .sorted(Comparator.comparingLong(TsKvEntry::getTs).reversed()) -// .limit(limit) -// .collect(Collectors.toList()); -// -// Map valueWithTs = limitedEntries.stream() -// .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue)); -// resultMap.put(key, valueWithTs); - -// return new LastRecordsArgumentEntry(limitedEntries); - return null; + arguments.forEach((key, argumentEntry) -> { + Argument argument = ctx.getArguments().get(key); + TreeMap tsRecords = ((LastRecordsArgumentEntry) argumentEntry).getTsRecords(); + if (tsRecords.size() > argument.getLimit()) { + tsRecords.pollFirstEntry(); + } + long necessaryIntervalTs = calculateIntervalStart(System.currentTimeMillis(), argument.getTimeWindow()); + tsRecords.entrySet().removeIf(tsRecord -> calculateIntervalStart(tsRecord.getKey(), argument.getTimeWindow()) < necessaryIntervalTs); + resultMap.put(key, tsRecords); }); Output output = ctx.getOutput(); return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), resultMap)); } + private long calculateIntervalStart(long ts, long interval) { + return (ts / interval) * interval; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 64eb409d3a..fc97141806 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -21,9 +21,7 @@ import lombok.Data; import net.objecthunter.exp4j.Expression; import net.objecthunter.exp4j.ExpressionBuilder; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -51,7 +49,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { customExpression.set(expr); } Map variables = new HashMap<>(); - this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(((KvEntry) v.getValue()).getValueAsString()))); + this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValue().toString()))); expr.setVariables(variables); double expressionResult = expr.evaluate(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 504d213748..e0db8c50fb 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -15,17 +15,29 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import lombok.AllArgsConstructor; import lombok.Data; -import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; @Data -@NoArgsConstructor -@AllArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { + private long ts; private Object value; + public SingleValueArgumentEntry() { + } + + public SingleValueArgumentEntry(KvEntry entry) { + if (entry instanceof TsKvEntry) { + this.ts = ((TsKvEntry) entry).getTs(); + } else if (entry instanceof AttributeKvEntry) { + this.ts = ((AttributeKvEntry) entry).getLastUpdateTs(); + } + this.value = entry.getValue(); + } + @Override public ArgumentType getType() { return ArgumentType.SINGLE_VALUE; From 2080b439a7e207ff186202c615f4cc161c010076 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 28 Nov 2024 11:28:51 +0200 Subject: [PATCH 11/21] added entity added/deleted events handling --- .../cf/CalculatedFieldExecutionService.java | 4 ++ ...efaultCalculatedFieldExecutionService.java | 46 ++++++++++++-- .../entitiy/EntityStateSourcingListener.java | 6 +- .../queue/DefaultTbClusterService.java | 61 +++++++++++++++++-- .../queue/DefaultTbCoreConsumerService.java | 27 ++++++++ .../server/cluster/TbClusterService.java | 2 + common/proto/src/main/proto/queue.proto | 13 ++++ .../server/dao/asset/BaseAssetService.java | 2 +- 8 files changed, 147 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 6d7cf0e741..a1202ec74b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -31,4 +31,8 @@ public interface CalculatedFieldExecutionService { void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); + void onEntityAdded(TransportProtos.EntityAddMsgProto proto, TbCallback callback); + + void onEntityDeleted(TransportProtos.EntityDeleteMsg proto, TbCallback callback); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 0d269f032f..3bb9be065d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -74,8 +74,8 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.LastRecordsCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; @@ -227,6 +227,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { + log.info("Received telemetry update msg: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> { CalculatedField calculatedField = calculatedFields.computeIfAbsent(id, cfId -> calculatedFieldService.findById(tenantId, id)); return new CalculatedFieldCtx(calculatedField, tbelInvokeService); @@ -247,7 +248,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); - log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) @@ -257,10 +257,44 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); }); - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, newProfileId) - .stream() - .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) - .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); + initializeStateForEntityByProfile(tenantId, entityId, newProfileId, callback); + } catch (Exception e) { + log.trace("Failed to process entity type update msg: [{}]", proto, e); + } + } + + @Override + public void onEntityAdded(TransportProtos.EntityAddMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + EntityId profileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB())); + log.info("Received EntityCreateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + + initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); + } catch (Exception e) { + log.trace("Failed to process entity type update msg: [{}]", proto, e); + } + } + + private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { + calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) + .stream() + .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) + .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); + } + + @Override + public void onEntityDeleted(TransportProtos.EntityDeleteMsg proto, TbCallback callback) { + try { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + log.info("Received EntityDeleteMsg for processing: entityId=[{}]", entityId); + List statesToRemove = states.keySet().stream() + .filter(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())) + .map(JacksonUtil::writeValueAsString) + .toList(); + states.keySet().removeIf(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())); + rocksDBService.deleteAll(statesToRemove); } catch (Exception e) { log.trace("Failed to process entity type update msg: [{}]", proto, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 154f5f4833..57293169a5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -146,7 +146,11 @@ public class EntityStateSourcingListener { log.debug("[{}][{}][{}] Handling entity deletion event: {}", tenantId, entityType, entityId, event); switch (entityType) { - case ASSET, ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE -> { + case ASSET -> { + Asset asset = (Asset) event.getEntity(); + tbClusterService.onAssetDeleted(tenantId, asset, null); + } + case ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); } case NOTIFICATION_REQUEST -> { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 58a16d9c0a..78a1f27a55 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -388,11 +388,26 @@ public class DefaultTbClusterService implements TbClusterService { public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) { DeviceId deviceId = device.getId(); gatewayNotificationsService.onDeviceDeleted(device); + handleEntityDelete(tenantId, deviceId, device.getDeviceProfileId()); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); } + @Override + public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { + AssetId assetId = asset.getId(); + handleEntityDelete(tenantId, assetId, asset.getAssetProfileId()); + broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); + } + + private void handleEntityDelete(TenantId tenantId, EntityId entityId, EntityId profileId) { + boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); + if (cfExistsByProfile) { + sendEntityDeletedEvent(tenantId, entityId); + } + } + @Override public void onDeviceAssignedToTenant(TenantId oldTenantId, Device device) { onDeviceDeleted(oldTenantId, device, null); @@ -613,11 +628,13 @@ public class DefaultTbClusterService implements TbClusterService { } boolean deviceTypeChanged = !device.getType().equals(old.getType()); if (deviceTypeChanged) { - handleProfileChange(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); + handleProfileUpdate(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); } if (deviceNameChanged || deviceTypeChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); } + } else { + handleEntityCreate(device.getTenantId(), device.getId(), device.getDeviceProfileId()); } broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), created, !created, false); @@ -631,16 +648,26 @@ public class DefaultTbClusterService implements TbClusterService { if (old != null) { boolean assetTypeChanged = !asset.getType().equals(old.getType()); if (assetTypeChanged) { - handleProfileChange(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); + handleProfileUpdate(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); } + } else { + handleEntityCreate(asset.getTenantId(), asset.getId(), asset.getAssetProfileId()); } broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } - private void handleProfileChange(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { - boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, oldProfileId); + private void handleProfileUpdate(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { + boolean cfExistsByOldProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, oldProfileId); + boolean cfExistsByNewProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, newProfileId); + if (cfExistsByOldProfile || cfExistsByNewProfile) { + sendEntityProfileUpdatedEvent(tenantId, entityId, oldProfileId, newProfileId); + } + } + + private void handleEntityCreate(TenantId tenantId, EntityId entityId, EntityId profileId) { + boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); if (cfExistsByProfile) { - sendEntityTypeUpdatedEvent(tenantId, entityId, oldProfileId, newProfileId); + sendEntityAddedEvent(tenantId, entityId, profileId); } } @@ -801,7 +828,7 @@ public class DefaultTbClusterService implements TbClusterService { pushMsgToCore(tenantId, calculatedFieldId, ToCoreMsg.newBuilder().setCalculatedFieldMsg(msg).build(), null); } - private void sendEntityTypeUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { + private void sendEntityProfileUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { TransportProtos.EntityProfileUpdateMsgProto.Builder builder = TransportProtos.EntityProfileUpdateMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); @@ -817,4 +844,26 @@ public class DefaultTbClusterService implements TbClusterService { pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityProfileUpdateMsg(msg).build(), null); } + private void sendEntityAddedEvent(TenantId tenantId, EntityId entityId, EntityId profileId) { + TransportProtos.EntityAddMsgProto.Builder builder = TransportProtos.EntityAddMsgProto.newBuilder(); + builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + builder.setEntityType(entityId.getEntityType().name()); + builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + builder.setEntityProfileType(profileId.getEntityType().name()); + builder.setProfileIdMSB(profileId.getId().getMostSignificantBits()); + builder.setProfileIdLSB(profileId.getId().getLeastSignificantBits()); + TransportProtos.EntityAddMsgProto msg = builder.build(); + pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityAddMsg(msg).build(), null); + } + + private void sendEntityDeletedEvent(TenantId tenantId, EntityId entityId) { + TransportProtos.EntityDeleteMsg.Builder builder = TransportProtos.EntityDeleteMsg.newBuilder(); + builder.setEntityType(entityId.getEntityType().name()); + builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityDeleteMsg(builder).build(), null); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 392e2637d1..d477c6d753 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -320,6 +320,10 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityAdded(entityCreateMsg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process entity create message for entityId [{}]", tenantId.getId(), entityId.getId(), t); + callback.onFailure(t); + }); + } + + private void forwardToCalculatedFieldService(TransportProtos.EntityDeleteMsg entityDeleteMsg, TbCallback callback) { + var entityId = EntityIdFactory.getByTypeAndUuid(entityDeleteMsg.getEntityType(), new UUID(entityDeleteMsg.getEntityIdMSB(), entityDeleteMsg.getEntityIdLSB())); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityDeleted(entityDeleteMsg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("Failed to process entity delete message for entity [{}]", entityId, t); + callback.onFailure(t); + }); + } + private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 131b50a52c..69334b774f 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -100,6 +100,8 @@ public interface TbClusterService extends TbQueueClusterService { void onAssetUpdated(Asset asset, Asset old); + void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback); + void onResourceChange(TbResourceInfo resource, TbQueueCallback callback); void onResourceDeleted(TbResourceInfo resource, TbQueueCallback callback); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index bf49a68c80..b1ca9b5e56 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -794,6 +794,17 @@ message EntityProfileUpdateMsgProto { int64 newProfileIdLSB = 10; } +message EntityAddMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + string entityType = 3; + int64 entityIdMSB = 4; + int64 entityIdLSB = 5; + string entityProfileType = 6; + int64 profileIdMSB = 7; + int64 profileIdLSB = 8; +} + //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; @@ -1527,6 +1538,8 @@ message ToCoreMsg { DeviceInactivityProto deviceInactivityMsg = 52; CalculatedFieldMsgProto calculatedFieldMsg = 53; EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; + EntityAddMsgProto entityAddMsg = 55; + EntityDeleteMsg entityDeleteMsg = 56; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index 7242cfde68..564da127ab 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -239,7 +239,7 @@ public class BaseAssetService extends AbstractCachedEntityService Date: Thu, 28 Nov 2024 16:07:55 +0200 Subject: [PATCH 12/21] implemented update telemetry for entity by profile --- .../cf/CalculatedFieldExecutionService.java | 3 +- ...efaultCalculatedFieldExecutionService.java | 4 +- .../DefaultTelemetrySubscriptionService.java | 61 +++++++++++++------ 3 files changed, 45 insertions(+), 23 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index a1202ec74b..b605ee909e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.cf; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -27,7 +28,7 @@ public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); + void onTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 3bb9be065d..d821f632fe 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -225,7 +225,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { log.info("Received telemetry update msg: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> { @@ -234,7 +234,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }); Map argumentValues = updatedTelemetry.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); - updateOrInitializeState(calculatedFieldCtx, calculatedFieldCtx.getEntityId(), argumentValues); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 76c1383f6a..f6c19eb078 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -33,8 +33,10 @@ import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -56,6 +58,8 @@ import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import java.util.ArrayList; @@ -85,6 +89,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final TbApiUsageStateService apiUsageStateService; private final CalculatedFieldService calculatedFieldService; private final CalculatedFieldExecutionService calculatedFieldExecutionService; + private final TbAssetProfileCache assetProfileCache; + private final TbDeviceProfileCache deviceProfileCache; private ExecutorService tsCallBackExecutor; @@ -97,7 +103,9 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer TbApiUsageReportClient apiUsageClient, TbApiUsageStateService apiUsageStateService, CalculatedFieldService calculatedFieldService, - CalculatedFieldExecutionService calculatedFieldExecutionService) { + CalculatedFieldExecutionService calculatedFieldExecutionService, + TbAssetProfileCache assetProfileCache, + TbDeviceProfileCache deviceProfileCache) { this.attrService = attrService; this.tsService = tsService; this.tbEntityViewService = tbEntityViewService; @@ -105,6 +113,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer this.apiUsageStateService = apiUsageStateService; this.calculatedFieldService = calculatedFieldService; this.calculatedFieldExecutionService = calculatedFieldExecutionService; + this.assetProfileCache = assetProfileCache; + this.deviceProfileCache = deviceProfileCache; } @PostConstruct @@ -201,8 +211,17 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } private void updateTelemetryInCalculatedFields(TenantId tenantId, EntityId entityId, List telemetry) { - if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { - List cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); + EntityType entityType = entityId.getEntityType(); + if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType)) { + EntityId profileId; + if (EntityType.ASSET.equals(entityType)) { + profileId = assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + } else { + profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); + } + List cfLinks = new ArrayList<>(); + cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); + cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, profileId)); if (!cfLinks.isEmpty()) { cfLinks.forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); @@ -211,34 +230,36 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer Map updatedTelemetry = telemetry.stream() .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) .collect(Collectors.toMap( - entry -> { - if (entry instanceof AttributeKvEntry) { - return attributes.entrySet().stream() - .filter(attr -> attr.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } else if (entry instanceof TsKvEntry) { - return timeSeries.entrySet().stream() - .filter(ts -> ts.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } - return entry.getKey(); - }, + entry -> getMappedKey(entry, attributes, timeSeries), entry -> entry, (v1, v2) -> v1 )); if (!updatedTelemetry.isEmpty()) { - calculatedFieldExecutionService.onTelemetryUpdate(tenantId, calculatedFieldId, updatedTelemetry); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, calculatedFieldId, updatedTelemetry); } }); } } } + private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { + if (entry instanceof AttributeKvEntry) { + return attributes.entrySet().stream() + .filter(attr -> attr.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } else if (entry instanceof TsKvEntry) { + return timeSeries.entrySet().stream() + .filter(ts -> ts.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } + return entry.getKey(); + } + private void addEntityViewCallback(TenantId tenantId, EntityId entityId, List ts) { if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { Futures.addCallback(this.tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId), From b910760b05c8597b1e44853f4eb928c7560b0865 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 29 Nov 2024 12:29:41 +0200 Subject: [PATCH 13/21] improved fetching cfs and states from db --- ...efaultCalculatedFieldExecutionService.java | 51 +++++++++---------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index d821f632fe..2147e0a1b7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -36,7 +36,6 @@ import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; @@ -81,7 +80,6 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -114,7 +112,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListeningExecutorService calculatedFieldCallbackExecutor; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); - private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); @@ -131,7 +128,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.submit(this::fetchCalculatedFields); } @PreDestroy @@ -176,7 +172,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } - CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); + CalculatedField cf = getOfFetchFromDb(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); @@ -184,11 +180,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return; } } - List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); if (cf != null) { EntityId entityId = cf.getEntityId(); - calculatedFields.put(calculatedFieldId, cf); - calculatedFieldLinks.put(calculatedFieldId, links); CalculatedFieldCtx calculatedFieldCtx = new CalculatedFieldCtx(cf, tbelInvokeService); calculatedFieldsCtx.put(calculatedFieldId, calculatedFieldCtx); switch (entityId.getEntityType()) { @@ -229,7 +222,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas try { log.info("Received telemetry update msg: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> { - CalculatedField calculatedField = calculatedFields.computeIfAbsent(id, cfId -> calculatedFieldService.findById(tenantId, id)); + CalculatedField calculatedField = getOfFetchFromDb(tenantId, id); return new CalculatedFieldCtx(calculatedField, tbelInvokeService); }); Map argumentValues = updatedTelemetry.entrySet().stream() @@ -300,14 +293,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private boolean onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); + private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { + CalculatedField oldCalculatedField = getOfFetchFromDb(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); boolean shouldReinit = true; - if (hasSignificantChanges(oldCalculatedField, newCalculatedField)) { - onCalculatedFieldDelete(newCalculatedField.getId(), callback); + if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { + onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); } else { - calculatedFields.put(newCalculatedField.getId(), newCalculatedField); - calculatedFieldsCtx.put(newCalculatedField.getId(), new CalculatedFieldCtx(newCalculatedField, tbelInvokeService)); + calculatedFields.put(updatedCalculatedField.getId(), updatedCalculatedField); + calculatedFieldsCtx.put(updatedCalculatedField.getId(), new CalculatedFieldCtx(updatedCalculatedField, tbelInvokeService)); callback.onSuccess(); shouldReinit = false; } @@ -316,12 +309,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { try { - calculatedFieldLinks.remove(calculatedFieldId); calculatedFields.remove(calculatedFieldId); calculatedFieldsCtx.remove(calculatedFieldId); - states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); List statesToRemove = states.keySet().stream() - .filter(ctxId -> !calculatedFields.containsKey(new CalculatedFieldId(ctxId.cfId()))) + .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) .map(JacksonUtil::writeValueAsString) .toList(); rocksDBService.deleteAll(statesToRemove); @@ -331,6 +323,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private CalculatedField getOfFetchFromDb(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + return calculatedFields.computeIfAbsent(calculatedFieldId, cfId -> calculatedFieldService.findById(tenantId, calculatedFieldId)); + } + private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { if (oldCalculatedField == null) { return true; @@ -346,15 +342,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; } - private void fetchCalculatedFields() { - PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); - cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); - PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); - cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class))); - states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); - } - private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { Map arguments = calculatedFieldCtx.getArguments(); Map argumentValues = new HashMap<>(); @@ -433,7 +420,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues) { CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(calculatedFieldCtx.getCfId().getId(), entityId.getId()); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctx -> new CalculatedFieldEntityCtx(entityCtxId, null)); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, this::fetchCalculatedFieldEntityState); CalculatedFieldState state = calculatedFieldEntityCtx.getState(); @@ -462,6 +449,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } + private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId) { + String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); + if (stateStr == null) { + return new CalculatedFieldEntityCtx(entityCtxId, null); + } + return JacksonUtil.fromString(rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)), CalculatedFieldEntityCtx.class); + } + private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { try { String type = calculatedFieldResult.getType(); From 894f4ead1b120d46809e6d03c326fa195fd69b0e Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 29 Nov 2024 17:31:56 +0200 Subject: [PATCH 14/21] implemented logic not to fetch tenant/customer telemetry for each field --- ...efaultCalculatedFieldExecutionService.java | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 2147e0a1b7..732d050a22 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -42,6 +42,7 @@ import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfig import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -80,6 +81,7 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -115,6 +117,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); + private final ConcurrentMap> tenantStorage = new ConcurrentHashMap<>(); + private final ConcurrentMap> customerStorage = new ConcurrentHashMap<>(); + private static final int MAX_LAST_RECORDS_VALUE = 1024; @Value("${calculatedField.initFetchPackSize:50000}") @@ -366,6 +371,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListenableFuture fetchArgumentValue(CalculatedFieldCtx calculatedFieldCtx, EntityId targetEntityId, Argument argument) { TenantId tenantId = calculatedFieldCtx.getTenantId(); EntityId argumentEntityId = argument.getEntityId(); + if (EntityType.TENANT.equals(argumentEntityId.getEntityType()) || EntityType.CUSTOMER.equals(argumentEntityId.getEntityType())) { + return fetchFromStorage(tenantId, argumentEntityId, argument); + } EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; @@ -375,6 +383,21 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return fetchKvEntry(tenantId, entityId, argument); } + private ListenableFuture fetchFromStorage(TenantId tenantId, EntityId entityId, Argument argument) { + List kvEntries; + if (EntityType.TENANT.equals(entityId.getEntityType())) { + kvEntries = tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); + } else { + kvEntries = customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); + } + for (KvEntry kvEntry : kvEntries) { + if (kvEntry.getKey().equals(argument.getKey())) { + return Futures.immediateFuture(ArgumentEntry.createSingleValueArgument(kvEntry)); + } + } + return fetchKvEntry(tenantId, entityId, argument); + } + private ListenableFuture fetchLastRecords(TenantId tenantId, EntityId entityId, Argument argument) { long currentTime = System.currentTimeMillis(); long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); @@ -403,7 +426,26 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldExecutor); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; - return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldExecutor); + return Futures.transform(kvEntryFuture, kvEntry -> { + if (EntityType.TENANT.equals(entityId.getEntityType()) || EntityType.CUSTOMER.equals(entityId.getEntityType())) { + updateStorage(tenantId, entityId, kvEntry); + } + return ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)); + }, calculatedFieldExecutor); + } + + private void updateStorage(TenantId tenantId, EntityId entityId, Optional kvEntry) { + if (kvEntry.isEmpty()) { + return; + } + List kvEntries = switch (entityId.getEntityType()) { + case TENANT -> tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); + case CUSTOMER -> customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); + default -> null; + }; + if (kvEntries != null && !kvEntries.contains(kvEntry.get())) { + kvEntries.add(kvEntry.get()); + } } private KvEntry createDefaultKvEntry(Argument argument) { From 19234df2b8e03d8d2af392faa32dfb44cc71bc77 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 5 Dec 2024 16:46:12 +0200 Subject: [PATCH 15/21] improved implementation of telemetry update --- ...efaultCalculatedFieldExecutionService.java | 60 ++++++++++++------- .../DefaultTelemetrySubscriptionService.java | 11 ++-- 2 files changed, 44 insertions(+), 27 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 732d050a22..4a1e987687 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -177,7 +177,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } - CalculatedField cf = getOfFetchFromDb(tenantId, calculatedFieldId); + CalculatedField cf = getOrFetchFromDb(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); @@ -226,14 +226,32 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { log.info("Received telemetry update msg: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> { - CalculatedField calculatedField = getOfFetchFromDb(tenantId, id); - return new CalculatedFieldCtx(calculatedField, tbelInvokeService); - }); + CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); - log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); + .collect(Collectors.toMap(Map.Entry::getKey, entry -> { + if (EntityType.TENANT.equals(entityId.getEntityType()) || EntityType.CUSTOMER.equals(entityId.getEntityType())) { + updateStorage(tenantId, entityId, Optional.of(entry.getValue())); + } + return ArgumentEntry.createSingleValueArgument(entry.getValue()); + })); + + EntityId cfEntityId = calculatedField.getEntityId(); + switch (cfEntityId.getEntityType()) { + case ASSET_PROFILE, DEVICE_PROFILE -> { + boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); + if (isCommonEntity) { + PageDataIterable entities = cfEntityId.getEntityType() == EntityType.ASSET_PROFILE + ? new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) cfEntityId, pageLink), initFetchPackSize) + : new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) cfEntityId, pageLink), initFetchPackSize); + entities.forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); + } else { + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); + } + } + default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues); + } + log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); } @@ -299,7 +317,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = getOfFetchFromDb(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); + CalculatedField oldCalculatedField = getOrFetchFromDb(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); boolean shouldReinit = true; if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); @@ -328,7 +346,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private CalculatedField getOfFetchFromDb(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + private CalculatedField getOrFetchFromDb(TenantId tenantId, CalculatedFieldId calculatedFieldId) { return calculatedFields.computeIfAbsent(calculatedFieldId, cfId -> calculatedFieldService.findById(tenantId, calculatedFieldId)); } @@ -435,17 +453,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void updateStorage(TenantId tenantId, EntityId entityId, Optional kvEntry) { - if (kvEntry.isEmpty()) { - return; - } - List kvEntries = switch (entityId.getEntityType()) { - case TENANT -> tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); - case CUSTOMER -> customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); - default -> null; - }; - if (kvEntries != null && !kvEntries.contains(kvEntry.get())) { - kvEntries.add(kvEntry.get()); - } + kvEntry.ifPresent(entry -> { + List kvEntries = switch (entityId.getEntityType()) { + case TENANT -> tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); + case CUSTOMER -> customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); + default -> null; + }; + if (kvEntries != null) { + kvEntries.removeIf(existingEntry -> existingEntry.getKey().equals(entry.getKey())); + kvEntries.add(entry); + } + }); } private KvEntry createDefaultKvEntry(Argument argument) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index f6c19eb078..ad64204e0f 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -212,16 +212,15 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private void updateTelemetryInCalculatedFields(TenantId tenantId, EntityId entityId, List telemetry) { EntityType entityType = entityId.getEntityType(); - if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType)) { - EntityId profileId; + if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType) || EntityType.CUSTOMER.equals(entityType) || EntityType.TENANT.equals(entityType)) { + EntityId profileId = null; if (EntityType.ASSET.equals(entityType)) { profileId = assetProfileCache.get(tenantId, (AssetId) entityId).getId(); - } else { + } else if (EntityType.DEVICE.equals(entityType)) { profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); } - List cfLinks = new ArrayList<>(); - cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); - cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, profileId)); + List cfLinks = new ArrayList<>(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); + Optional.ofNullable(profileId).ifPresent(id -> cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, id))); if (!cfLinks.isEmpty()) { cfLinks.forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); From dfeb0a7dbaf30b28529e27449f0fac9015f066f8 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 6 Dec 2024 11:11:24 +0200 Subject: [PATCH 16/21] made executeScrioptAsync method flexible --- .../cf/ctx/state/CalculatedFieldScriptEngine.java | 4 ++-- .../cf/ctx/state/CalculatedFieldTbelScriptEngine.java | 11 ++++------- .../cf/ctx/state/ScriptCalculatedFieldState.java | 3 ++- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java index 212d971398..779f52c5d6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java @@ -21,9 +21,9 @@ import java.util.Map; public interface CalculatedFieldScriptEngine { - ListenableFuture executeScriptAsync(Map arguments); + ListenableFuture executeScriptAsync(Object[] args); - ListenableFuture> executeToMapAsync(Map arguments); + ListenableFuture> executeToMapAsync(Object[] args); ListenableFuture> executeToMapTransform(Object result); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java index 363eefd44e..7ac032573f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -19,11 +19,9 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.ScriptType; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; import javax.script.ScriptException; import java.util.Map; @@ -53,9 +51,8 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng } @Override - public ListenableFuture executeScriptAsync(Map arguments) { - log.trace("execute script async, arguments {}", arguments); - Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); + public ListenableFuture executeScriptAsync(Object[] args) { + log.trace("Executing script async, args {}", args); return Futures.transformAsync(tbelInvokeService.invokeScript(tenantId, null, this.scriptId, args), o -> { try { @@ -73,8 +70,8 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng } @Override - public ListenableFuture> executeToMapAsync(Map arguments) { - return Futures.transformAsync(executeScriptAsync(arguments), this::executeToMapTransform, MoreExecutors.directExecutor()); + public ListenableFuture> executeToMapAsync(Object[] args) { + return Futures.transformAsync(executeScriptAsync(args), this::executeToMapTransform, MoreExecutors.directExecutor()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 16905930d3..99befa6e65 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -39,7 +39,8 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { Output output = ctx.getOutput(); if (isValid(ctx.getArguments())) { - ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(this.arguments); + Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); + ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); return Futures.transform(resultFuture, result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), MoreExecutors.directExecutor() From 303851015035cc3715190b5fc708ad6da173e807 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 9 Dec 2024 11:04:05 +0200 Subject: [PATCH 17/21] added implementation for handling profile entities --- .../cf/CalculatedFieldExecutionService.java | 4 +- ...efaultCalculatedFieldExecutionService.java | 96 ++++++++++--------- .../queue/DefaultTbClusterService.java | 22 ++--- .../queue/DefaultTbCoreConsumerService.java | 27 ++---- common/proto/src/main/proto/queue.proto | 7 +- 5 files changed, 71 insertions(+), 85 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index b605ee909e..5a85529f6b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -32,8 +32,6 @@ public interface CalculatedFieldExecutionService { void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); - void onEntityAdded(TransportProtos.EntityAddMsgProto proto, TbCallback callback); - - void onEntityDeleted(TransportProtos.EntityDeleteMsg proto, TbCallback callback); + void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 4a1e987687..7ded80d013 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -39,11 +39,9 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -83,6 +81,7 @@ import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -117,6 +116,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); + private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); private final ConcurrentMap> tenantStorage = new ConcurrentHashMap<>(); private final ConcurrentMap> customerStorage = new ConcurrentHashMap<>(); @@ -194,17 +194,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); initializeStateForEntity(calculatedFieldCtx, entityId, callback); } - case ASSET_PROFILE -> { - log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); - PageDataIterable assetIds = new PageDataIterable<>(pageLink -> - assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - assetIds.forEach(assetId -> initializeStateForEntity(calculatedFieldCtx, assetId, callback)); - } - case DEVICE_PROFILE -> { - log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); - PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> - deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - deviceIds.forEach(deviceId -> initializeStateForEntity(calculatedFieldCtx, deviceId, callback)); + case ASSET_PROFILE, DEVICE_PROFILE -> { + log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); + getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(assetId -> initializeStateForEntity(calculatedFieldCtx, assetId, callback)); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -241,10 +233,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); if (isCommonEntity) { - PageDataIterable entities = cfEntityId.getEntityType() == EntityType.ASSET_PROFILE - ? new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) cfEntityId, pageLink), initFetchPackSize) - : new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) cfEntityId, pageLink), initFetchPackSize); - entities.forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); + getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); } else { updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); } @@ -266,6 +255,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + profileEntities.get(oldProfileId).remove(entityId); + profileEntities.computeIfAbsent(newProfileId, id -> new HashSet<>()).add(entityId); + calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> { CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); @@ -280,39 +272,28 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onEntityAdded(TransportProtos.EntityAddMsgProto proto, TbCallback callback) { + public void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); EntityId profileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB())); - log.info("Received EntityCreateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - - initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); - } catch (Exception e) { - log.trace("Failed to process entity type update msg: [{}]", proto, e); - } - } - - private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) - .stream() - .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) - .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); - } - - @Override - public void onEntityDeleted(TransportProtos.EntityDeleteMsg proto, TbCallback callback) { - try { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - log.info("Received EntityDeleteMsg for processing: entityId=[{}]", entityId); - List statesToRemove = states.keySet().stream() - .filter(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())) - .map(JacksonUtil::writeValueAsString) - .toList(); - states.keySet().removeIf(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())); - rocksDBService.deleteAll(statesToRemove); + log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + if (proto.getDeleted()) { + log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); + profileEntities.get(profileId).remove(entityId); + List statesToRemove = states.keySet().stream() + .filter(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())) + .map(JacksonUtil::writeValueAsString) + .toList(); + states.keySet().removeIf(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())); + rocksDBService.deleteAll(statesToRemove); + } else { + log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); + profileEntities.computeIfAbsent(profileId, id -> new HashSet<>()).add(entityId); + initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); + } } catch (Exception e) { - log.trace("Failed to process entity type update msg: [{}]", proto, e); + log.trace("Failed to process profile entity msg: [{}]", proto, e); } } @@ -350,6 +331,24 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return calculatedFields.computeIfAbsent(calculatedFieldId, cfId -> calculatedFieldService.findById(tenantId, calculatedFieldId)); } + private Set getOrFetchFromDBProfileEntities(TenantId tenantId, EntityId entityProfileId) { + return switch (entityProfileId.getEntityType()) { + case ASSET_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { + Set assetIds = new HashSet<>(); + (new PageDataIterable<>(pageLink -> + assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) profileId, pageLink), initFetchPackSize)).forEach(assetIds::add); + return assetIds; + }); + case DEVICE_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { + Set deviceIds = new HashSet<>(); + (new PageDataIterable<>(pageLink -> + deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityProfileId, pageLink), initFetchPackSize)).forEach(deviceIds::add); + return deviceIds; + }); + default -> throw new IllegalArgumentException("Entity type should be ASSET_PROFILE or DEVICE_PROFILE."); + }; + } + private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { if (oldCalculatedField == null) { return true; @@ -365,6 +364,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; } + private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { + calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) + .stream() + .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) + .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); + } + private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { Map arguments = calculatedFieldCtx.getArguments(); Map argumentValues = new HashMap<>(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 78a1f27a55..a706f943c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -404,7 +404,7 @@ public class DefaultTbClusterService implements TbClusterService { private void handleEntityDelete(TenantId tenantId, EntityId entityId, EntityId profileId) { boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); if (cfExistsByProfile) { - sendEntityDeletedEvent(tenantId, entityId); + sendProfileEntityEvent(tenantId, entityId, profileId, false, true); } } @@ -667,7 +667,7 @@ public class DefaultTbClusterService implements TbClusterService { private void handleEntityCreate(TenantId tenantId, EntityId entityId, EntityId profileId) { boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); if (cfExistsByProfile) { - sendEntityAddedEvent(tenantId, entityId, profileId); + sendProfileEntityEvent(tenantId, entityId, profileId, true, false); } } @@ -844,8 +844,8 @@ public class DefaultTbClusterService implements TbClusterService { pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityProfileUpdateMsg(msg).build(), null); } - private void sendEntityAddedEvent(TenantId tenantId, EntityId entityId, EntityId profileId) { - TransportProtos.EntityAddMsgProto.Builder builder = TransportProtos.EntityAddMsgProto.newBuilder(); + private void sendProfileEntityEvent(TenantId tenantId, EntityId entityId, EntityId profileId, boolean added, boolean deleted) { + TransportProtos.ProfileEntityMsgProto.Builder builder = TransportProtos.ProfileEntityMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); builder.setEntityType(entityId.getEntityType().name()); @@ -854,16 +854,10 @@ public class DefaultTbClusterService implements TbClusterService { builder.setEntityProfileType(profileId.getEntityType().name()); builder.setProfileIdMSB(profileId.getId().getMostSignificantBits()); builder.setProfileIdLSB(profileId.getId().getLeastSignificantBits()); - TransportProtos.EntityAddMsgProto msg = builder.build(); - pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityAddMsg(msg).build(), null); - } - - private void sendEntityDeletedEvent(TenantId tenantId, EntityId entityId) { - TransportProtos.EntityDeleteMsg.Builder builder = TransportProtos.EntityDeleteMsg.newBuilder(); - builder.setEntityType(entityId.getEntityType().name()); - builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); - builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityDeleteMsg(builder).build(), null); + builder.setAdded(added); + builder.setDeleted(deleted); + TransportProtos.ProfileEntityMsgProto msg = builder.build(); + pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setProfileEntityMsg(msg).build(), null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index d477c6d753..6cf42cb893 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -320,10 +320,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityAdded(entityCreateMsg, callback)); + private void forwardToCalculatedFieldService(TransportProtos.ProfileEntityMsgProto profileEntityMsgProto, TbCallback callback) { + var tenantId = toTenantId(profileEntityMsgProto.getTenantIdMSB(), profileEntityMsgProto.getTenantIdLSB()); + var entityId = EntityIdFactory.getByTypeAndUuid(profileEntityMsgProto.getEntityType(), new UUID(profileEntityMsgProto.getEntityIdMSB(), profileEntityMsgProto.getEntityIdLSB())); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onProfileEntityMsg(profileEntityMsgProto, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { - log.warn("[{}] Failed to process entity create message for entityId [{}]", tenantId.getId(), entityId.getId(), t); - callback.onFailure(t); - }); - } - - private void forwardToCalculatedFieldService(TransportProtos.EntityDeleteMsg entityDeleteMsg, TbCallback callback) { - var entityId = EntityIdFactory.getByTypeAndUuid(entityDeleteMsg.getEntityType(), new UUID(entityDeleteMsg.getEntityIdMSB(), entityDeleteMsg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityDeleted(entityDeleteMsg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("Failed to process entity delete message for entity [{}]", entityId, t); + log.warn("[{}] Failed to process profile entity message for entityId [{}]", tenantId.getId(), entityId.getId(), t); callback.onFailure(t); }); } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index b1ca9b5e56..76da257b32 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -794,7 +794,7 @@ message EntityProfileUpdateMsgProto { int64 newProfileIdLSB = 10; } -message EntityAddMsgProto { +message ProfileEntityMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; string entityType = 3; @@ -803,6 +803,8 @@ message EntityAddMsgProto { string entityProfileType = 6; int64 profileIdMSB = 7; int64 profileIdLSB = 8; + bool added = 9; + bool deleted = 10; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. @@ -1538,8 +1540,7 @@ message ToCoreMsg { DeviceInactivityProto deviceInactivityMsg = 52; CalculatedFieldMsgProto calculatedFieldMsg = 53; EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; - EntityAddMsgProto entityAddMsg = 55; - EntityDeleteMsg entityDeleteMsg = 56; + ProfileEntityMsgProto profileEntityMsg = 55; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ From 4fed2cd416b13b7f05159ba893f47dcd788fb13d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 9 Dec 2024 13:04:41 +0200 Subject: [PATCH 18/21] improved ability to fetch common arguments for profile cfs --- ...efaultCalculatedFieldExecutionService.java | 120 +++++++++--------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 7ded80d013..f2629381b2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -41,7 +41,6 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -89,7 +88,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; @@ -117,8 +116,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final ConcurrentMap states = new ConcurrentHashMap<>(); private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); - private final ConcurrentMap> tenantStorage = new ConcurrentHashMap<>(); - private final ConcurrentMap> customerStorage = new ConcurrentHashMap<>(); private static final int MAX_LAST_RECORDS_VALUE = 1024; @@ -196,7 +193,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } case ASSET_PROFILE, DEVICE_PROFILE -> { log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); - getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(assetId -> initializeStateForEntity(calculatedFieldCtx, assetId, callback)); + fetchCommonArguments(calculatedFieldCtx, callback, commonArguments -> { + getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(assetId -> { + initializeStateForEntity(calculatedFieldCtx, assetId, commonArguments, callback); + }); + }); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -221,12 +222,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> { - if (EntityType.TENANT.equals(entityId.getEntityType()) || EntityType.CUSTOMER.equals(entityId.getEntityType())) { - updateStorage(tenantId, entityId, Optional.of(entry.getValue())); - } - return ArgumentEntry.createSingleValueArgument(entry.getValue()); - })); + .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); EntityId cfEntityId = calculatedField.getEntityId(); switch (cfEntityId.getEntityType()) { @@ -371,33 +367,71 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } - private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { - Map arguments = calculatedFieldCtx.getArguments(); + private void fetchCommonArguments(CalculatedFieldCtx calculatedFieldCtx, TbCallback callback, Consumer> onComplete) { Map argumentValues = new HashMap<>(); - AtomicInteger remaining = new AtomicInteger(arguments.size()); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedFieldCtx, entityId, argument), new FutureCallback<>() { + List> futures = new ArrayList<>(); + + calculatedFieldCtx.getArguments().forEach((key, argument) -> { + if (!EntityType.DEVICE_PROFILE.equals(argument.getEntityId().getEntityType()) && + !EntityType.ASSET_PROFILE.equals(argument.getEntityId().getEntityType())) { + futures.add(Futures.transform(fetchKvEntry(calculatedFieldCtx.getTenantId(), argument.getEntityId(), argument), + result -> { + argumentValues.put(key, result); + return result; + }, calculatedFieldCallbackExecutor)); + } + }); + + Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { @Override - public void onSuccess(ArgumentEntry result) { - argumentValues.put(key, result); - if (remaining.decrementAndGet() == 0) { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); - } + public void onSuccess(List results) { + onComplete.accept(argumentValues); + } + + @Override + public void onFailure(Throwable t) { + log.error("Failed to fetch common arguments", t); + callback.onFailure(t); + } + }, calculatedFieldCallbackExecutor); + } + + private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { + initializeStateForEntity(calculatedFieldCtx, entityId, new HashMap<>(), callback); + } + + private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map commonArguments, TbCallback callback) { + Map argumentValues = new HashMap<>(commonArguments); + List> futures = new ArrayList<>(); + + calculatedFieldCtx.getArguments().forEach((key, argument) -> { + if (!commonArguments.containsKey(key)) { + futures.add(Futures.transform(fetchArgumentValue(calculatedFieldCtx, entityId, argument), + result -> { + argumentValues.put(key, result); + return result; + }, calculatedFieldCallbackExecutor)); + } + }); + + Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { + @Override + public void onSuccess(List results) { + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); + callback.onSuccess(); } @Override public void onFailure(Throwable t) { - log.warn("Failed to initialize state for entity: [{}]", entityId, t); + log.error("Failed to initialize state for entity: [{}]", entityId, t); callback.onFailure(t); } - }, calculatedFieldCallbackExecutor)); + }, calculatedFieldCallbackExecutor); } private ListenableFuture fetchArgumentValue(CalculatedFieldCtx calculatedFieldCtx, EntityId targetEntityId, Argument argument) { TenantId tenantId = calculatedFieldCtx.getTenantId(); EntityId argumentEntityId = argument.getEntityId(); - if (EntityType.TENANT.equals(argumentEntityId.getEntityType()) || EntityType.CUSTOMER.equals(argumentEntityId.getEntityType())) { - return fetchFromStorage(tenantId, argumentEntityId, argument); - } EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; @@ -407,21 +441,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return fetchKvEntry(tenantId, entityId, argument); } - private ListenableFuture fetchFromStorage(TenantId tenantId, EntityId entityId, Argument argument) { - List kvEntries; - if (EntityType.TENANT.equals(entityId.getEntityType())) { - kvEntries = tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); - } else { - kvEntries = customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); - } - for (KvEntry kvEntry : kvEntries) { - if (kvEntry.getKey().equals(argument.getKey())) { - return Futures.immediateFuture(ArgumentEntry.createSingleValueArgument(kvEntry)); - } - } - return fetchKvEntry(tenantId, entityId, argument); - } - private ListenableFuture fetchLastRecords(TenantId tenantId, EntityId entityId, Argument argument) { long currentTime = System.currentTimeMillis(); long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); @@ -450,26 +469,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldExecutor); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; - return Futures.transform(kvEntryFuture, kvEntry -> { - if (EntityType.TENANT.equals(entityId.getEntityType()) || EntityType.CUSTOMER.equals(entityId.getEntityType())) { - updateStorage(tenantId, entityId, kvEntry); - } - return ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)); - }, calculatedFieldExecutor); - } - - private void updateStorage(TenantId tenantId, EntityId entityId, Optional kvEntry) { - kvEntry.ifPresent(entry -> { - List kvEntries = switch (entityId.getEntityType()) { - case TENANT -> tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); - case CUSTOMER -> customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); - default -> null; - }; - if (kvEntries != null) { - kvEntries.removeIf(existingEntry -> existingEntry.getKey().equals(entry.getKey())); - kvEntries.add(entry); - } - }); + return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { From 86569c312e96e297ad1de55b31c681bf32c36a28 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 10 Dec 2024 12:21:13 +0200 Subject: [PATCH 19/21] added ability to perform calculations on the last records --- .../cf/ctx/state/CalculatedFieldCtx.java | 2 +- .../ctx/state/LastRecordsArgumentEntry.java | 2 +- .../LastRecordsCalculatedFieldState.java | 37 ++++++++++--------- .../ctx/state/ScriptCalculatedFieldState.java | 2 +- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 10395a0d5d..b436e0421e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -51,7 +51,7 @@ public class CalculatedFieldCtx { this.output = configuration.getOutput(); this.expression = configuration.getExpression(); this.tbelInvokeService = tbelInvokeService; - if (CalculatedFieldType.SCRIPT.equals(calculatedField.getType())) { + if (!CalculatedFieldType.SIMPLE.equals(calculatedField.getType())) { this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java index 93fabd3cb8..8b672d4de9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java @@ -37,7 +37,7 @@ public class LastRecordsArgumentEntry implements ArgumentEntry { @JsonIgnore @Override public Object getValue() { - return tsRecords.values(); + return tsRecords; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java index dd69790236..fbcf72fda9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java @@ -17,13 +17,13 @@ package org.thingsboard.server.service.cf.ctx.state; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.Data; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.service.cf.CalculatedFieldResult; -import java.util.HashMap; import java.util.Map; import java.util.TreeMap; @@ -56,23 +56,24 @@ public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - Map resultMap = new HashMap<>(); - arguments.forEach((key, argumentEntry) -> { - Argument argument = ctx.getArguments().get(key); - TreeMap tsRecords = ((LastRecordsArgumentEntry) argumentEntry).getTsRecords(); - if (tsRecords.size() > argument.getLimit()) { - tsRecords.pollFirstEntry(); - } - long necessaryIntervalTs = calculateIntervalStart(System.currentTimeMillis(), argument.getTimeWindow()); - tsRecords.entrySet().removeIf(tsRecord -> calculateIntervalStart(tsRecord.getKey(), argument.getTimeWindow()) < necessaryIntervalTs); - resultMap.put(key, tsRecords); - }); - Output output = ctx.getOutput(); - return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), resultMap)); - } - - private long calculateIntervalStart(long ts, long interval) { - return (ts / interval) * interval; + if (isValid(ctx.getArguments())) { + arguments.forEach((key, argumentEntry) -> { + Argument argument = ctx.getArguments().get(key); + TreeMap tsRecords = ((LastRecordsArgumentEntry) argumentEntry).getTsRecords(); + if (tsRecords.size() > argument.getLimit()) { + tsRecords.pollFirstEntry(); + } + tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); + }); + Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); + ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); + Output output = ctx.getOutput(); + return Futures.transform(resultFuture, + result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), + MoreExecutors.directExecutor() + ); + } + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 99befa6e65..ba050d3b71 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -37,10 +37,10 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - Output output = ctx.getOutput(); if (isValid(ctx.getArguments())) { Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); + Output output = ctx.getOutput(); return Futures.transform(resultFuture, result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), MoreExecutors.directExecutor() From a73affea23683c902a5ee45ad3b569c0463fd41b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 12 Dec 2024 17:04:24 +0200 Subject: [PATCH 20/21] removed CF type LAST_RECORDS and implemented this functionality as a script(added new argument type TS_ROLLING) --- ...efaultCalculatedFieldExecutionService.java | 49 ++++++------ .../service/cf/ctx/state/ArgumentEntry.java | 6 +- .../service/cf/ctx/state/ArgumentType.java | 2 +- .../ctx/state/BaseCalculatedFieldState.java | 26 +++--- .../cf/ctx/state/CalculatedFieldState.java | 2 - .../LastRecordsCalculatedFieldState.java | 79 ------------------- .../ctx/state/ScriptCalculatedFieldState.java | 14 +++- .../ctx/state/SimpleCalculatedFieldState.java | 2 +- ...Entry.java => TsRollingArgumentEntry.java} | 4 +- .../common/data/cf/CalculatedFieldType.java | 2 +- .../BaseCalculatedFieldConfiguration.java | 4 +- .../CalculatedFieldConfiguration.java | 3 +- ...stRecordsCalculatedFieldConfiguration.java | 39 --------- .../dao/model/sql/CalculatedFieldEntity.java | 2 - ...efaultNativeCalculatedFieldRepository.java | 2 - 15 files changed, 64 insertions(+), 172 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/{LastRecordsArgumentEntry.java => TsRollingArgumentEntry.java} (91%) delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f2629381b2..050c2d7e73 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -73,12 +73,12 @@ import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; -import org.thingsboard.server.service.cf.ctx.state.LastRecordsCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -435,41 +435,41 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; - if (CalculatedFieldType.LAST_RECORDS.equals(calculatedFieldCtx.getCfType())) { - return fetchLastRecords(tenantId, entityId, argument); - } return fetchKvEntry(tenantId, entityId, argument); } - private ListenableFuture fetchLastRecords(TenantId tenantId, EntityId entityId, Argument argument) { + private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { + return switch (argument.getType()) { + case "TS_ROLLING" -> fetchTsRolling(tenantId, entityId, argument); + case "ATTRIBUTE" -> transformSingleValueArgument( + Futures.transform( + attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), + result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + calculatedFieldCallbackExecutor) + ); + case "TS_LATEST" -> transformSingleValueArgument( + Futures.transform( + timeseriesService.findLatest(tenantId, entityId, argument.getKey()), + result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + calculatedFieldCallbackExecutor)); + default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); + }; + } + + private ListenableFuture fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument) { long currentTime = System.currentTimeMillis(); long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); long startTs = currentTime - timeWindow; int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); - ListenableFuture> lastRecordsFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); + ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); - return Futures.transform(lastRecordsFuture, ArgumentEntry::createLastRecordsArgument, calculatedFieldExecutor); + return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? ArgumentEntry.createTsRollingArgument(Collections.emptyList()) : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); } - private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { - ListenableFuture> kvEntryFuture = switch (argument.getType()) { - case "ATTRIBUTES" -> Futures.transform( - attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), - result -> result.or(() -> Optional.of( - new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)) - )), - MoreExecutors.directExecutor()); - case "TIME_SERIES" -> Futures.transform( - timeseriesService.findLatest(tenantId, entityId, argument.getKey()), - result -> result.or(() -> Optional.of( - new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)) - )), - calculatedFieldExecutor); - default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); - }; - return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldExecutor); + private ListenableFuture transformSingleValueArgument(ListenableFuture> kvEntryFuture) { + return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldCallbackExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { @@ -547,7 +547,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return switch (calculatedFieldType) { case SIMPLE -> new SimpleCalculatedFieldState(); case SCRIPT -> new ScriptCalculatedFieldState(); - case LAST_RECORDS -> new LastRecordsCalculatedFieldState(); }; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 6a62c0b1e3..f70d614123 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -32,7 +32,7 @@ import java.util.stream.Collectors; ) @JsonSubTypes({ @JsonSubTypes.Type(value = SingleValueArgumentEntry.class, name = "SINGLE_VALUE"), - @JsonSubTypes.Type(value = LastRecordsArgumentEntry.class, name = "LAST_RECORDS") + @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING") }) public interface ArgumentEntry { @@ -45,8 +45,8 @@ public interface ArgumentEntry { return new SingleValueArgumentEntry(kvEntry); } - static ArgumentEntry createLastRecordsArgument(List kvEntries) { - return new LastRecordsArgumentEntry(kvEntries.stream(). + static ArgumentEntry createTsRollingArgument(List kvEntries) { + return new TsRollingArgumentEntry(kvEntries.stream(). collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue, (oldValue, newValue) -> newValue, TreeMap::new))); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java index f2f0eac60d..360529a7e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java @@ -16,5 +16,5 @@ package org.thingsboard.server.service.cf.ctx.state; public enum ArgumentType { - SINGLE_VALUE, LAST_RECORDS + SINGLE_VALUE, TS_ROLLING } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 54df89e757..59b007a420 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf.ctx.state; import java.util.HashMap; import java.util.Map; -import java.util.stream.Collectors; public abstract class BaseCalculatedFieldState implements CalculatedFieldState { @@ -36,15 +35,22 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { if (arguments == null) { arguments = new HashMap<>(); } - arguments.putAll( - argumentValues.entrySet().stream() - .peek(entry -> { - if (entry.getValue() instanceof LastRecordsArgumentEntry) { - throw new IllegalArgumentException("Last records argument entry is not allowed for single calculated field state"); - } - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) - ); + argumentValues.forEach((key, argumentEntry) -> { + ArgumentEntry existingArgumentEntry = arguments.get(key); + if (existingArgumentEntry != null) { + if (existingArgumentEntry instanceof SingleValueArgumentEntry) { + arguments.put(key, argumentEntry); + } else if (existingArgumentEntry instanceof TsRollingArgumentEntry existingTsRollingArgumentEntry) { + if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { + existingTsRollingArgumentEntry.getTsRecords().putAll(tsRollingArgumentEntry.getTsRecords()); + } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + existingTsRollingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); + } + } + } else { + arguments.put(key, argumentEntry); + } + }); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index f0882cbea4..a5ac6b2c47 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.Map; @@ -34,7 +33,6 @@ import java.util.Map; @JsonSubTypes({ @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE"), @JsonSubTypes.Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT"), - @JsonSubTypes.Type(value = LastRecordsCalculatedFieldState.class, name = "LAST_RECORDS") }) public interface CalculatedFieldState { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java deleted file mode 100644 index fbcf72fda9..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cf.ctx.state; - -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; -import lombok.Data; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.service.cf.CalculatedFieldResult; - -import java.util.Map; -import java.util.TreeMap; - -@Data -public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { - - public LastRecordsCalculatedFieldState() { - } - - @Override - public CalculatedFieldType getType() { - return CalculatedFieldType.LAST_RECORDS; - } - - @Override - public void initState(Map argumentValues) { - if (arguments == null) { - arguments = new TreeMap<>(); - } - argumentValues.forEach((key, argumentEntry) -> { - LastRecordsArgumentEntry existingArgumentEntry = (LastRecordsArgumentEntry) - arguments.computeIfAbsent(key, k -> new LastRecordsArgumentEntry(new TreeMap<>())); - if (argumentEntry instanceof LastRecordsArgumentEntry lastRecordsArgumentEntry) { - existingArgumentEntry.getTsRecords().putAll(lastRecordsArgumentEntry.getTsRecords()); - } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - existingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); - } - }); - } - - @Override - public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - if (isValid(ctx.getArguments())) { - arguments.forEach((key, argumentEntry) -> { - Argument argument = ctx.getArguments().get(key); - TreeMap tsRecords = ((LastRecordsArgumentEntry) argumentEntry).getTsRecords(); - if (tsRecords.size() > argument.getLimit()) { - tsRecords.pollFirstEntry(); - } - tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); - }); - Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); - ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); - Output output = ctx.getOutput(); - return Futures.transform(resultFuture, - result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), - MoreExecutors.directExecutor() - ); - } - return null; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index ba050d3b71..b9b98f9c5e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -21,10 +21,12 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.Map; +import java.util.TreeMap; @Data @Slf4j @@ -38,6 +40,16 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { if (isValid(ctx.getArguments())) { + arguments.forEach((key, argumentEntry) -> { + if (argumentEntry instanceof TsRollingArgumentEntry) { + Argument argument = ctx.getArguments().get(key); + TreeMap tsRecords = ((TsRollingArgumentEntry) argumentEntry).getTsRecords(); + if (tsRecords.size() > argument.getLimit()) { + tsRecords.pollFirstEntry(); + } + tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); + } + }); Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); Output output = ctx.getOutput(); @@ -46,7 +58,7 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { MoreExecutors.directExecutor() ); } - return null; + return Futures.immediateFuture(null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index fc97141806..491419b40a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -57,7 +57,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { Output output = ctx.getOutput(); return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), Map.of(output.getName(), expressionResult))); } - return null; + return Futures.immediateFuture(null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 8b672d4de9..1166da113d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -25,13 +25,13 @@ import java.util.TreeMap; @Data @NoArgsConstructor @AllArgsConstructor -public class LastRecordsArgumentEntry implements ArgumentEntry { +public class TsRollingArgumentEntry implements ArgumentEntry { private TreeMap tsRecords; @Override public ArgumentType getType() { - return ArgumentType.LAST_RECORDS; + return ArgumentType.TS_ROLLING; } @JsonIgnore diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java index 63b6d8d1dd..89173b35b9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -17,6 +17,6 @@ package org.thingsboard.server.common.data.cf; public enum CalculatedFieldType { - SIMPLE, SCRIPT, LAST_RECORDS + SIMPLE, SCRIPT } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index f311fb737b..ac36991a61 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -69,10 +69,10 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel Argument argument = entry.getValue(); if (argument.getEntityId().equals(entityId)) { switch (argument.getType()) { - case "ATTRIBUTES": + case "ATTRIBUTE": linkConfiguration.getAttributes().put(entry.getKey(), argument.getKey()); break; - case "TIME_SERIES": + case "TS_LATEST", "TS_ROLLING": linkConfiguration.getTimeSeries().put(entry.getKey(), argument.getKey()); break; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 15f7a82c40..5c428bd628 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -35,8 +35,7 @@ import java.util.UUID; ) @JsonSubTypes({ @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE"), - @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT"), - @JsonSubTypes.Type(value = LastRecordsCalculatedFieldConfiguration.class, name = "LAST_RECORDS") + @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT") }) public interface CalculatedFieldConfiguration { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java deleted file mode 100644 index c3f5804227..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.cf.configuration; - -import com.fasterxml.jackson.databind.JsonNode; -import lombok.Data; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; - -import java.util.UUID; - -@Data -public class LastRecordsCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { - - public LastRecordsCalculatedFieldConfiguration() { - } - - public LastRecordsCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - super(config, entityType, entityId); - } - - @Override - public CalculatedFieldType getType() { - return CalculatedFieldType.LAST_RECORDS; - } -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index b06676f70b..6aaaf05836 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -26,7 +26,6 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.LastRecordsCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -124,7 +123,6 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem return switch (CalculatedFieldType.valueOf(type)) { case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - case LAST_RECORDS -> new LastRecordsCalculatedFieldConfiguration(config, entityType, entityId); }; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index 2acd4d75c6..a5a2743f26 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -29,7 +29,6 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.LastRecordsCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -140,7 +139,6 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF return switch (type) { case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - case LAST_RECORDS -> new LastRecordsCalculatedFieldConfiguration(config, entityType, entityId); }; } From 4668bece03229c34aa74c335bd6602ddf9885c71 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 13 Dec 2024 11:14:56 +0200 Subject: [PATCH 21/21] fixed tests --- .../server/controller/CalculatedFieldControllerTest.java | 4 ++-- .../org/thingsboard/server/dao/service/AssetServiceTest.java | 4 ++-- .../server/dao/service/CalculatedFieldServiceTest.java | 4 ++-- .../thingsboard/server/dao/service/CustomerServiceTest.java | 4 ++-- .../org/thingsboard/server/dao/service/DeviceServiceTest.java | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 77ca268d12..ba1dfb1fec 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -140,7 +140,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { Argument argument = new Argument(); argument.setEntityId(referencedEntityId); - argument.setType("TIME_SERIES"); + argument.setType("TS_LATEST"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -149,7 +149,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { Output output = new Output(); output.setName("output"); - output.setType("TIME_SERIES"); + output.setType("TS_LATEST"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 87f2cb1f45..b0870f3dc5 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -884,7 +884,7 @@ public class AssetServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedAsset.getId()); - argument.setType("TIME_SERIES"); + argument.setType("TS_LATEST"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -893,7 +893,7 @@ public class AssetServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TIME_SERIES"); + output.setType("TS_LATEST"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 77ed026b1d..9a1719e715 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -153,7 +153,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(referencedEntityId); - argument.setType("TIME_SERIES"); + argument.setType("TS_LATEST"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -162,7 +162,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TIME_SERIES"); + output.setType("TS_LATEST"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 94c8440057..6671e0e821 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -379,7 +379,7 @@ public class CustomerServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedCustomer.getId()); - argument.setType("TIME_SERIES"); + argument.setType("TS_LATEST"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -388,7 +388,7 @@ public class CustomerServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TIME_SERIES"); + output.setType("TS_LATEST"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index d5394a3494..f2f8686bc3 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -1222,7 +1222,7 @@ public class DeviceServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(device.getId()); - argument.setType("TIME_SERIES"); + argument.setType("TS_LATEST"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -1231,7 +1231,7 @@ public class DeviceServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TIME_SERIES"); + output.setType("TS_LATEST"); config.setOutput(output);