From 19f6f323260347e5af7eb0317dbe168dd9cb8e52 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 16 Dec 2024 12:47:55 +0200 Subject: [PATCH 01/14] moved logic when telemetry update to main cf service --- .../cf/CalculatedFieldExecutionService.java | 5 +- .../service/cf/CalculatedFieldResult.java | 6 +- ...efaultCalculatedFieldExecutionService.java | 149 +++++++++++++----- .../cf/ctx/CalculatedFieldEntityCtx.java | 5 +- .../service/cf/ctx/state/ArgumentEntry.java | 2 + .../ctx/state/BaseCalculatedFieldState.java | 22 ++- .../cf/ctx/state/CalculatedFieldCtx.java | 2 +- .../cf/ctx/state/CalculatedFieldState.java | 7 +- .../ctx/state/ScriptCalculatedFieldState.java | 35 ++-- .../ctx/state/SimpleCalculatedFieldState.java | 37 ++--- .../ctx/state/SingleValueArgumentEntry.java | 9 +- .../cf/ctx/state/TsRollingArgumentEntry.java | 4 + .../DefaultTelemetrySubscriptionService.java | 78 +-------- 13 files changed, 186 insertions(+), 175 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 5a85529f6b..302e4b2511 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,20 +15,19 @@ */ package org.thingsboard.server.service.cf; -import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos; -import java.util.Map; +import java.util.List; public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - void onTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); + void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List telemetry); void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index 1f8a06c8fa..e8ea318bf6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -23,9 +23,9 @@ import java.util.Map; @Data public final class CalculatedFieldResult { - private String type; - private AttributeScope scope; - private Map resultMap; + private final String type; + private final AttributeScope scope; + private final Map resultMap; public CalculatedFieldResult(String type, AttributeScope scope, Map resultMap) { this.type = type; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 050c2d7e73..c1a86ac36f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -36,16 +36,20 @@ import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; @@ -76,6 +80,8 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; import java.util.Collections; @@ -102,6 +108,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final CalculatedFieldService calculatedFieldService; private final AssetService assetService; private final DeviceService deviceService; + private final TbAssetProfileCache assetProfileCache; + private final TbDeviceProfileCache deviceProfileCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; private final RocksDBService rocksDBService; @@ -112,6 +120,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListeningExecutorService calculatedFieldCallbackExecutor; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); @@ -130,6 +139,16 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); + scheduledExecutor.submit(this::fetchCalculatedFields); + } + + private void fetchCalculatedFields() { + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); + PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); + cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class))); + states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); } @PreDestroy @@ -216,30 +235,77 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List telemetry) { try { - log.info("Received telemetry update msg: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); - CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); - Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); - - EntityId cfEntityId = calculatedField.getEntityId(); - switch (cfEntityId.getEntityType()) { - case ASSET_PROFILE, DEVICE_PROFILE -> { - boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); - if (isCommonEntity) { - getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); - } else { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); - } + EntityType entityType = entityId.getEntityType(); + if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType) || EntityType.CUSTOMER.equals(entityType) || EntityType.TENANT.equals(entityType)) { + EntityId profileId = null; + if (EntityType.ASSET.equals(entityType)) { + profileId = assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + } else if (EntityType.DEVICE.equals(entityType)) { + profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); } - default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues); + List cfLinks = new ArrayList<>(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); + Optional.ofNullable(profileId).ifPresent(id -> cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, id))); + cfLinks.forEach(link -> { + CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); + Map attributes = link.getConfiguration().getAttributes(); + Map timeSeries = link.getConfiguration().getTimeSeries(); + Map updatedTelemetry = telemetry.stream() + .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) + .collect(Collectors.toMap( + entry -> getMappedKey(entry, attributes, timeSeries), + entry -> entry, + (v1, v2) -> v1 + )); + + if (!updatedTelemetry.isEmpty()) { + executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, updatedTelemetry); + } + }); } - log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { - log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); + log.trace("Failed to update telemetry entityId: [{}]", entityId, e); + } + } + + private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); + CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); + Map argumentValues = updatedTelemetry.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); + + EntityId cfEntityId = calculatedField.getEntityId(); + switch (cfEntityId.getEntityType()) { + case ASSET_PROFILE, DEVICE_PROFILE -> { + boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); + if (isCommonEntity) { + getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); + } else { + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); + } + } + default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues); + } + log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); + } + + private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { + if (entry instanceof AttributeKvEntry) { + return attributes.entrySet().stream() + .filter(attr -> attr.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } else if (entry instanceof TsKvEntry) { + return timeSeries.entrySet().stream() + .filter(ts -> ts.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); } + return entry.getKey(); } @Override @@ -493,26 +559,29 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (state == null) { state = createStateByType(calculatedFieldCtx.getCfType()); } - state.initState(argumentValues); - calculatedFieldEntityCtx.setState(state); - states.put(entityCtxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - - ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); - Futures.addCallback(resultFuture, new FutureCallback<>() { - @Override - public void onSuccess(CalculatedFieldResult result) { - if (result != null) { - pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); - } - } + if (state.updateState(argumentValues)) { + calculatedFieldEntityCtx.setState(state); + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + + boolean allArgsPresent = calculatedFieldCtx.getArguments().keySet().containsAll(state.getArguments().keySet()); + if (allArgsPresent) { + ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); + Futures.addCallback(resultFuture, new FutureCallback<>() { + @Override + public void onSuccess(CalculatedFieldResult result) { + if (result != null) { + pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); + } + } - @Override - public void onFailure(Throwable t) { - log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); + @Override + public void onFailure(Throwable t) { + log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); + } + }, MoreExecutors.directExecutor()); } - }, MoreExecutors.directExecutor()); - + } } private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId) { @@ -520,14 +589,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (stateStr == null) { return new CalculatedFieldEntityCtx(entityCtxId, null); } - return JacksonUtil.fromString(rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)), CalculatedFieldEntityCtx.class); + return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); } private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { try { String type = calculatedFieldResult.getType(); - TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; - TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; + TbMsgType msgType = "ATTRIBUTE".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; + TbMsgMetaData md = "ATTRIBUTE".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(payload)); clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java index 7a8384b6bf..e6dc021951 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java @@ -16,17 +16,16 @@ package org.thingsboard.server.service.cf.ctx; import lombok.Data; +import lombok.NoArgsConstructor; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @Data +@NoArgsConstructor public class CalculatedFieldEntityCtx { private CalculatedFieldEntityCtxId id; private CalculatedFieldState state; - public CalculatedFieldEntityCtx() { - } - public CalculatedFieldEntityCtx(CalculatedFieldEntityCtxId id, CalculatedFieldState state) { this.id = id; this.state = state; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index f70d614123..78222244c9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -41,6 +41,8 @@ public interface ArgumentEntry { Object getValue(); + boolean hasUpdatedValue(ArgumentEntry entry); + static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { return new SingleValueArgumentEntry(kvEntry); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 59b007a420..ae6fc9033a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf.ctx.state; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; public abstract class BaseCalculatedFieldState implements CalculatedFieldState { @@ -31,26 +32,39 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { } @Override - public void initState(Map argumentValues) { + public boolean updateState(Map argumentValues) { if (arguments == null) { arguments = new HashMap<>(); } + AtomicBoolean stateUpdated = new AtomicBoolean(false); argumentValues.forEach((key, argumentEntry) -> { ArgumentEntry existingArgumentEntry = arguments.get(key); if (existingArgumentEntry != null) { if (existingArgumentEntry instanceof SingleValueArgumentEntry) { - arguments.put(key, argumentEntry); + if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { + arguments.put(key, argumentEntry); + stateUpdated.set(true); + } } else if (existingArgumentEntry instanceof TsRollingArgumentEntry existingTsRollingArgumentEntry) { if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { - existingTsRollingArgumentEntry.getTsRecords().putAll(tsRollingArgumentEntry.getTsRecords()); + if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { + existingTsRollingArgumentEntry.getTsRecords().putAll(tsRollingArgumentEntry.getTsRecords()); + stateUpdated.set(true); + } } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - existingTsRollingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); + if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { + existingTsRollingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); + stateUpdated.set(true); + } + } } } else { arguments.put(key, argumentEntry); + stateUpdated.set(true); } }); + return stateUpdated.get(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index b436e0421e..2cd5c68144 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -35,7 +35,7 @@ public class CalculatedFieldCtx { private TenantId tenantId; private EntityId entityId; private CalculatedFieldType cfType; - private Map arguments; + private final Map arguments; private Output output; private String expression; private TbelInvokeService tbelInvokeService; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index a5ac6b2c47..3c4a680df1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.Map; @@ -41,11 +40,7 @@ public interface CalculatedFieldState { Map getArguments(); - default boolean isValid(Map arguments) { - return getArguments().keySet().containsAll(arguments.keySet()); - } - - void initState(Map argumentValues); + boolean updateState(Map argumentValues); ListenableFuture performCalculation(CalculatedFieldCtx ctx); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index b9b98f9c5e..87429050de 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -39,26 +39,23 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - if (isValid(ctx.getArguments())) { - arguments.forEach((key, argumentEntry) -> { - if (argumentEntry instanceof TsRollingArgumentEntry) { - Argument argument = ctx.getArguments().get(key); - TreeMap tsRecords = ((TsRollingArgumentEntry) argumentEntry).getTsRecords(); - if (tsRecords.size() > argument.getLimit()) { - tsRecords.pollFirstEntry(); - } - tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); + arguments.forEach((key, argumentEntry) -> { + if (argumentEntry instanceof TsRollingArgumentEntry) { + Argument argument = ctx.getArguments().get(key); + TreeMap tsRecords = ((TsRollingArgumentEntry) argumentEntry).getTsRecords(); + if (tsRecords.size() > argument.getLimit()) { + tsRecords.pollFirstEntry(); } - }); - Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); - ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); - Output output = ctx.getOutput(); - return Futures.transform(resultFuture, - result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), - MoreExecutors.directExecutor() - ); - } - return Futures.immediateFuture(null); + tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); + } + }); + Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); + ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); + Output output = ctx.getOutput(); + return Futures.transform(resultFuture, + result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), + MoreExecutors.directExecutor() + ); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 491419b40a..e16d310b3e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -37,27 +37,24 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - if (isValid(ctx.getArguments())) { - String expression = ctx.getExpression(); - ThreadLocal customExpression = new ThreadLocal<>(); - var expr = customExpression.get(); - if (expr == null) { - expr = new ExpressionBuilder(expression) - .implicitMultiplication(true) - .variables(this.arguments.keySet()) - .build(); - customExpression.set(expr); - } - Map variables = new HashMap<>(); - this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValue().toString()))); - expr.setVariables(variables); - - double expressionResult = expr.evaluate(); - - Output output = ctx.getOutput(); - return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), Map.of(output.getName(), expressionResult))); + String expression = ctx.getExpression(); + ThreadLocal customExpression = new ThreadLocal<>(); + var expr = customExpression.get(); + if (expr == null) { + expr = new ExpressionBuilder(expression) + .implicitMultiplication(true) + .variables(this.arguments.keySet()) + .build(); + customExpression.set(expr); } - return Futures.immediateFuture(null); + Map variables = new HashMap<>(); + this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValue().toString()))); + expr.setVariables(variables); + + double expressionResult = expr.evaluate(); + + Output output = ctx.getOutput(); + return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), Map.of(output.getName(), expressionResult))); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index e0db8c50fb..e6cb24b970 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -16,19 +16,18 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.Data; +import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; @Data +@NoArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { private long ts; private Object value; - public SingleValueArgumentEntry() { - } - public SingleValueArgumentEntry(KvEntry entry) { if (entry instanceof TsKvEntry) { this.ts = ((TsKvEntry) entry).getTs(); @@ -48,4 +47,8 @@ public class SingleValueArgumentEntry implements ArgumentEntry { return value; } + @Override + public boolean hasUpdatedValue(ArgumentEntry entry) { + return this.ts != ((SingleValueArgumentEntry) entry).getTs(); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 1166da113d..104d0ae90c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -40,4 +40,8 @@ public class TsRollingArgumentEntry implements ArgumentEntry { return tsRecords; } + @Override + public boolean hasUpdatedValue(ArgumentEntry entry) { + return !tsRecords.containsKey(((SingleValueArgumentEntry) entry).getTs()); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index ad64204e0f..b94b319e71 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -32,11 +32,7 @@ import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -44,7 +40,6 @@ import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -52,14 +47,11 @@ import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.stats.TbApiUsageReportClient; import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; -import org.thingsboard.server.service.profile.TbAssetProfileCache; -import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import java.util.ArrayList; @@ -73,7 +65,6 @@ import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.stream.Collectors; /** * Created by ashvayka on 27.03.18. @@ -87,11 +78,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final TbEntityViewService tbEntityViewService; private final TbApiUsageReportClient apiUsageClient; private final TbApiUsageStateService apiUsageStateService; - private final CalculatedFieldService calculatedFieldService; private final CalculatedFieldExecutionService calculatedFieldExecutionService; - private final TbAssetProfileCache assetProfileCache; - private final TbDeviceProfileCache deviceProfileCache; - private ExecutorService tsCallBackExecutor; @Value("${sql.ts.value_no_xss_validation:false}") @@ -102,19 +89,13 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Lazy TbEntityViewService tbEntityViewService, TbApiUsageReportClient apiUsageClient, TbApiUsageStateService apiUsageStateService, - CalculatedFieldService calculatedFieldService, - CalculatedFieldExecutionService calculatedFieldExecutionService, - TbAssetProfileCache assetProfileCache, - TbDeviceProfileCache deviceProfileCache) { + CalculatedFieldExecutionService calculatedFieldExecutionService) { this.attrService = attrService; this.tsService = tsService; this.tbEntityViewService = tbEntityViewService; this.apiUsageClient = apiUsageClient; this.apiUsageStateService = apiUsageStateService; - this.calculatedFieldService = calculatedFieldService; this.calculatedFieldExecutionService = calculatedFieldExecutionService; - this.assetProfileCache = assetProfileCache; - this.deviceProfileCache = deviceProfileCache; } @PostConstruct @@ -201,7 +182,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer addMainCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); addEntityViewCallback(tenantId, entityId, ts); - updateTelemetryInCalculatedFields(tenantId, entityId, ts); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, ts); } private void saveWithoutLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { @@ -210,55 +191,6 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); } - private void updateTelemetryInCalculatedFields(TenantId tenantId, EntityId entityId, List telemetry) { - EntityType entityType = entityId.getEntityType(); - if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType) || EntityType.CUSTOMER.equals(entityType) || EntityType.TENANT.equals(entityType)) { - EntityId profileId = null; - if (EntityType.ASSET.equals(entityType)) { - profileId = assetProfileCache.get(tenantId, (AssetId) entityId).getId(); - } else if (EntityType.DEVICE.equals(entityType)) { - profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); - } - List cfLinks = new ArrayList<>(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); - Optional.ofNullable(profileId).ifPresent(id -> cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, id))); - if (!cfLinks.isEmpty()) { - cfLinks.forEach(link -> { - CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - Map attributes = link.getConfiguration().getAttributes(); - Map timeSeries = link.getConfiguration().getTimeSeries(); - Map updatedTelemetry = telemetry.stream() - .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) - .collect(Collectors.toMap( - entry -> getMappedKey(entry, attributes, timeSeries), - entry -> entry, - (v1, v2) -> v1 - )); - - if (!updatedTelemetry.isEmpty()) { - calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, calculatedFieldId, updatedTelemetry); - } - }); - } - } - } - - private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { - if (entry instanceof AttributeKvEntry) { - return attributes.entrySet().stream() - .filter(attr -> attr.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } else if (entry instanceof TsKvEntry) { - return timeSeries.entrySet().stream() - .filter(ts -> ts.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } - return entry.getKey(); - } - private void addEntityViewCallback(TenantId tenantId, EntityId entityId, List ts) { if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { Futures.addCallback(this.tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId), @@ -335,7 +267,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice)); - updateTelemetryInCalculatedFields(tenantId, entityId, attributes); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, attributes); } @Override @@ -343,7 +275,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope.name(), attributes, notifyDevice)); - updateTelemetryInCalculatedFields(tenantId, entityId, attributes); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, attributes); } @Override @@ -357,7 +289,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = tsService.saveLatest(tenantId, entityId, ts); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); - updateTelemetryInCalculatedFields(tenantId, entityId, ts); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, ts); } @Override From 481753b8f0e0b15f3174f65ee18fca33eb0a7568 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 17 Dec 2024 13:41:23 +0200 Subject: [PATCH 02/14] added restriction for number of cfs/arguments/ts rolling values --- .../service/cf/ctx/state/ArgumentEntry.java | 3 ++ .../ctx/state/BaseCalculatedFieldState.java | 9 ++--- .../ctx/state/SingleValueArgumentEntry.java | 8 ++++ .../cf/ctx/state/TsRollingArgumentEntry.java | 37 +++++++++++++++++-- .../cf/DefaultTbCalculatedFieldService.java | 18 +++++++++ 5 files changed, 67 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 78222244c9..ba7e094f77 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -52,4 +52,7 @@ public interface ArgumentEntry { collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue, (oldValue, newValue) -> newValue, TreeMap::new))); } + @JsonIgnore + ArgumentEntry copy(); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index ae6fc9033a..b73ac51798 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -42,25 +42,24 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { if (existingArgumentEntry != null) { if (existingArgumentEntry instanceof SingleValueArgumentEntry) { if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - arguments.put(key, argumentEntry); + arguments.put(key, argumentEntry.copy()); stateUpdated.set(true); } } else if (existingArgumentEntry instanceof TsRollingArgumentEntry existingTsRollingArgumentEntry) { if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - existingTsRollingArgumentEntry.getTsRecords().putAll(tsRollingArgumentEntry.getTsRecords()); + existingTsRollingArgumentEntry.addAllTsRecords(tsRollingArgumentEntry.getTsRecords()); stateUpdated.set(true); } } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - existingTsRollingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); + existingTsRollingArgumentEntry.addTsRecord(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); stateUpdated.set(true); } - } } } else { - arguments.put(key, argumentEntry); + arguments.put(key, argumentEntry.copy()); stateUpdated.set(true); } }); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index e6cb24b970..81a57580db 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -23,6 +24,7 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; @Data @NoArgsConstructor +@AllArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { private long ts; @@ -51,4 +53,10 @@ public class SingleValueArgumentEntry implements ArgumentEntry { public boolean hasUpdatedValue(ArgumentEntry entry) { return this.ts != ((SingleValueArgumentEntry) entry).getTs(); } + + @Override + public ArgumentEntry copy() { + return new SingleValueArgumentEntry(this.ts, this.value); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 104d0ae90c..49aae15ac1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -16,18 +16,26 @@ package org.thingsboard.server.service.cf.ctx.state; import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.math.NumberUtils; +import java.util.Map; import java.util.TreeMap; @Data @NoArgsConstructor -@AllArgsConstructor +@Slf4j public class TsRollingArgumentEntry implements ArgumentEntry { - private TreeMap tsRecords; + private static final int MAX_ROLLING_ARGUMENT_ENTRY_SIZE = 1000; + + private TreeMap tsRecords = new TreeMap<>(); + + public TsRollingArgumentEntry(TreeMap tsRecords) { + addAllTsRecords(tsRecords); + } @Override public ArgumentType getType() { @@ -44,4 +52,27 @@ public class TsRollingArgumentEntry implements ArgumentEntry { public boolean hasUpdatedValue(ArgumentEntry entry) { return !tsRecords.containsKey(((SingleValueArgumentEntry) entry).getTs()); } + + @Override + public ArgumentEntry copy() { + return new TsRollingArgumentEntry(new TreeMap<>(tsRecords)); + } + + public void addTsRecord(Long key, Object value) { + if (NumberUtils.isParsable(value.toString())) { + tsRecords.put(key, value); + if (tsRecords.size() > MAX_ROLLING_ARGUMENT_ENTRY_SIZE) { + tsRecords.pollFirstEntry(); + } + } else { + log.warn("Argument type 'TS_ROLLING' only supports numeric values."); + } + } + + public void addAllTsRecords(Map newRecords) { + for (Map.Entry entry : newRecords.entrySet()) { + addTsRecord(entry.getKey(), entry.getValue()); + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 4d28ff55ac..2e6e975636 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -46,6 +46,9 @@ import static org.thingsboard.server.dao.service.Validator.validateEntityId; @RequiredArgsConstructor public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { + private static final int MAX_ARGUMENT_SIZE = 10; + private static final int MAX_CALCULATED_FIELD_NUMBER = 10; + private final CalculatedFieldService calculatedFieldService; @Override @@ -53,7 +56,9 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp ActionType actionType = calculatedField.getId() == null ? ActionType.ADDED : ActionType.UPDATED; TenantId tenantId = calculatedField.getTenantId(); try { + checkCalculatedFieldNumber(tenantId, calculatedField.getEntityId()); checkEntityExistence(tenantId, calculatedField.getEntityId()); + checkArgumentSize(calculatedField.getConfiguration()); checkReferencedEntities(calculatedField.getConfiguration(), user); CalculatedField savedCalculatedField = checkNotNull(calculatedFieldService.save(calculatedField)); logEntityActionService.logEntityAction(tenantId, savedCalculatedField.getId(), savedCalculatedField, actionType, user); @@ -105,6 +110,19 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } + private void checkArgumentSize(CalculatedFieldConfiguration calculatedFieldConfig) { + if (calculatedFieldConfig.getArguments().size() > MAX_ARGUMENT_SIZE) { + throw new IllegalArgumentException("Too many arguments: " + calculatedFieldConfig.getArguments().size() + ". Max number of argument is " + MAX_ARGUMENT_SIZE); + } + } + + private void checkCalculatedFieldNumber(TenantId tenantId, EntityId entityId) { + int numberOfCalculatedFieldsByEntityId = calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, entityId).size(); + if (numberOfCalculatedFieldsByEntityId >= MAX_CALCULATED_FIELD_NUMBER) { + throw new IllegalArgumentException("Max number of calculated fields for entity is " + MAX_CALCULATED_FIELD_NUMBER); + } + } + private & HasTenantId, I extends EntityId> E findEntity(TenantId tenantId, EntityId entityId) { return switch (entityId.getEntityType()) { case TENANT, CUSTOMER, ASSET, DEVICE -> (E) entityService.fetchEntity(tenantId, entityId).orElse(null); From f929de42097c4929cae2d8970ea45dfced2b12b0 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 18 Dec 2024 13:12:39 +0200 Subject: [PATCH 03/14] onAddedPartitions() impl --- ...efaultCalculatedFieldExecutionService.java | 65 +++++++++++++++++-- 1 file changed, 60 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index c1a86ac36f..7e5ea5b5ec 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Lists; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -63,6 +64,7 @@ import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.asset.AssetService; @@ -173,13 +175,66 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override protected Map>> onAddedPartitions(Set addedPartitions) { - // TODO: implementation for cluster mode - return Map.of(); + var result = new HashMap>>(); + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + Map> tpiCalculatedFieldMap = new HashMap<>(); + + for (CalculatedField cf : cfs) { + TopicPartitionInfo tpi; + try { + tpi = partitionService.resolve(ServiceType.TB_CORE, cf.getTenantId(), cf.getId()); + } catch (Exception e) { + log.warn("Failed to resolve partition for CalculatedField [{}], tenant [{}]. Reason: {}", + cf.getId(), cf.getTenantId(), e.getMessage()); + continue; + } + if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId().getId()))) { + tpiCalculatedFieldMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(cf); + } + } + + for (var entry : tpiCalculatedFieldMap.entrySet()) { + for (List partition : Lists.partition(entry.getValue(), 1000)) { + log.info("[{}] Submit task for CalculatedFields: {}", entry.getKey(), partition.size()); + var future = calculatedFieldExecutor.submit(() -> { + try { + for (CalculatedField cf : partition) { + if (EntityType.ASSET_PROFILE.equals(cf.getEntityId().getEntityType()) || EntityType.DEVICE_PROFILE.equals(cf.getEntityId().getEntityType())) { + getOrFetchFromDBProfileEntities(cf.getTenantId(), cf.getEntityId()) + .forEach(entityId -> restoreState(cf, entityId)); + } else { + restoreState(cf, cf.getEntityId()); + } + } + } catch (Throwable t) { + log.error("Unexpected exception while restoring CalculatedField states", t); + throw t; + } + }); + result.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(future); + } + } + return result; + } + + private void restoreState(CalculatedField cf, EntityId entityId) { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cf.getId().getId(), entityId.getId()); + String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); + + if (storedState != null) { + CalculatedFieldEntityCtx restoredCtx = JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class); + calculatedFieldsCtx.putIfAbsent(cf.getId(), new CalculatedFieldCtx(cf, tbelInvokeService)); + states.put(ctxId, restoredCtx); + log.info("Restored state for CalculatedField [{}]", cf.getId()); + } else { + log.warn("No state found for CalculatedField [{}], entity [{}].", cf.getId(), entityId); + } } @Override protected void cleanupEntityOnPartitionRemoval(CalculatedFieldId entityId) { - // TODO: implementation for cluster mode + calculatedFields.remove(entityId); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(entityId.getId())); } @Override @@ -213,8 +268,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); fetchCommonArguments(calculatedFieldCtx, callback, commonArguments -> { - getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(assetId -> { - initializeStateForEntity(calculatedFieldCtx, assetId, commonArguments, callback); + getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(targetEntityId -> { + initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArguments, callback); }); }); } From 5c5dc474cbc2ba899e9aa61106a211981e122dec Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 19 Dec 2024 17:12:59 +0200 Subject: [PATCH 04/14] implemented partitioning --- .../cf/CalculatedFieldExecutionService.java | 4 +- ...efaultCalculatedFieldExecutionService.java | 115 +++++++++++++++--- .../queue/DefaultTbCoreConsumerService.java | 16 ++- common/proto/src/main/proto/queue.proto | 12 ++ 4 files changed, 125 insertions(+), 22 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 302e4b2511..966b5d7277 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -29,7 +29,9 @@ public interface CalculatedFieldExecutionService { void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List telemetry); - void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); + void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback); + + void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 7e5ea5b5ec..f2a09d0faf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -98,6 +98,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.thingsboard.server.common.data.DataConstants.SCOPE; @@ -184,7 +185,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas try { tpi = partitionService.resolve(ServiceType.TB_CORE, cf.getTenantId(), cf.getId()); } catch (Exception e) { - log.warn("Failed to resolve partition for CalculatedField [{}], tenant [{}]. Reason: {}", + log.warn("Failed to resolve partition for CalculatedField [{}], tenant id [{}]. Reason: {}", cf.getId(), cf.getTenantId(), e.getMessage()); continue; } @@ -233,8 +234,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override protected void cleanupEntityOnPartitionRemoval(CalculatedFieldId entityId) { - calculatedFields.remove(entityId); - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(entityId.getId())); + cleanupEntity(entityId); + } + + private void cleanupEntity(CalculatedFieldId calculatedFieldId) { + calculatedFields.remove(calculatedFieldId); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); } @Override @@ -245,7 +250,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); if (proto.getDeleted()) { log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); - onCalculatedFieldDelete(calculatedFieldId, callback); + onCalculatedFieldDelete(tenantId, calculatedFieldId, callback); callback.onSuccess(); } CalculatedField cf = getOrFetchFromDb(tenantId, calculatedFieldId); @@ -364,7 +369,34 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { + public void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + String state = proto.getState(); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = state.isEmpty() ? JacksonUtil.fromString(state, CalculatedFieldEntityCtx.class) : null; + + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + if (tpi.isMyPartition()) { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId()); + if (calculatedFieldEntityCtx != null) { + states.put(ctxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), state); + } else { + states.remove(ctxId); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + } + } else { + log.debug("[{}] Calculated Field belongs to external partition {}", calculatedFieldId, tpi.getFullTopicName()); + } + } catch (Exception e) { + log.trace("Failed to process calculated field update state msg: [{}]", proto, e); + } + } + + @Override + public void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); @@ -377,9 +409,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); + if (tpi.isMyPartition()) { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); + states.remove(ctxId); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + } else { + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, null); + } }); initializeStateForEntityByProfile(tenantId, entityId, newProfileId, callback); @@ -398,12 +435,22 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (proto.getDeleted()) { log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); profileEntities.get(profileId).remove(entityId); - List statesToRemove = states.keySet().stream() - .filter(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())) - .map(JacksonUtil::writeValueAsString) - .toList(); - states.keySet().removeIf(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())); - rocksDBService.deleteAll(statesToRemove); + List calculatedFieldIds = Stream.concat( + calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId).stream() + .map(CalculatedFieldLink::getCalculatedFieldId), + calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, profileId).stream() + .map(CalculatedFieldLink::getCalculatedFieldId) + ).toList(); + calculatedFieldIds.forEach(cfId -> { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); + if (tpi.isMyPartition()) { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); + states.remove(ctxId); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + } else { + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, null); + } + }); } else { log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); profileEntities.computeIfAbsent(profileId, id -> new HashSet<>()).add(entityId); @@ -414,11 +461,26 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, CalculatedFieldState calculatedFieldState) { + TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = TransportProtos.CalculatedFieldStateMsgProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + if (calculatedFieldState != null) { + msgBuilder.setState(JacksonUtil.writeValueAsString(calculatedFieldState)); + } + clusterService.pushMsgToCore(tenantId, entityId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); + } + private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { CalculatedField oldCalculatedField = getOrFetchFromDb(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); boolean shouldReinit = true; if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { - onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); + onCalculatedFieldDelete(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId(), callback); } else { calculatedFields.put(updatedCalculatedField.getId(), updatedCalculatedField); calculatedFieldsCtx.put(updatedCalculatedField.getId(), new CalculatedFieldCtx(updatedCalculatedField, tbelInvokeService)); @@ -428,8 +490,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return shouldReinit; } - private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { + private void onCalculatedFieldDelete(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbCallback callback) { try { + cleanupEntity(calculatedFieldId); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + Set calculatedFieldIds = partitionedEntities.get(tpi); + if (calculatedFieldIds != null) { + calculatedFieldIds.remove(calculatedFieldId); + } calculatedFields.remove(calculatedFieldId); calculatedFieldsCtx.remove(calculatedFieldId); states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); @@ -606,7 +674,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues) { - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(calculatedFieldCtx.getCfId().getId(), entityId.getId()); + CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, this::fetchCalculatedFieldEntityState); CalculatedFieldState state = calculatedFieldEntityCtx.getState(); @@ -616,8 +685,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } if (state.updateState(argumentValues)) { calculatedFieldEntityCtx.setState(state); - states.put(entityCtxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + TenantId tenantId = calculatedFieldCtx.getTenantId(); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldCtx.getCfId()); + if (tpi.isMyPartition()) { + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + } else { + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, state); + } boolean allArgsPresent = calculatedFieldCtx.getArguments().keySet().containsAll(state.getArguments().keySet()); if (allArgsPresent) { @@ -626,7 +701,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onSuccess(CalculatedFieldResult result) { if (result != null) { - pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); + pushMsgToRuleEngine(tenantId, entityId, result); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 6cf42cb893..a686184b8c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -322,6 +322,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChanged(profileUpdateMsg, callback)); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChangedMsg(profileUpdateMsg, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { @@ -708,6 +710,18 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldStateMsg(calculatedFieldStateMsgProto, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process calculated field state message for entityId [{}]", tenantId.getId(), calculatedFieldId.getId(), t); + callback.onFailure(t); + }); + } + private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 76da257b32..153394525d 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -807,6 +807,17 @@ message ProfileEntityMsgProto { bool deleted = 10; } +message CalculatedFieldStateMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; + string entityType = 5; + int64 entityIdMSB = 6; + int64 entityIdLSB = 7; + string state = 8; +} + //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; @@ -1541,6 +1552,7 @@ message ToCoreMsg { CalculatedFieldMsgProto calculatedFieldMsg = 53; EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; ProfileEntityMsgProto profileEntityMsg = 55; + CalculatedFieldStateMsgProto calculatedFieldStateMsg = 56; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ From 2a41c8b45123e43fc34def1a954e098298c7339e Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 20 Dec 2024 16:16:58 +0200 Subject: [PATCH 05/14] implemented logic to fetch telemetry if states were not fetched --- ...efaultCalculatedFieldExecutionService.java | 171 ++++++++++-------- .../ctx/state/SingleValueArgumentEntry.java | 10 + .../cf/ctx/state/TsRollingArgumentEntry.java | 13 +- 3 files changed, 118 insertions(+), 76 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f2a09d0faf..1128a75008 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -81,12 +81,13 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -97,6 +98,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -200,11 +202,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas var future = calculatedFieldExecutor.submit(() -> { try { for (CalculatedField cf : partition) { - if (EntityType.ASSET_PROFILE.equals(cf.getEntityId().getEntityType()) || EntityType.DEVICE_PROFILE.equals(cf.getEntityId().getEntityType())) { - getOrFetchFromDBProfileEntities(cf.getTenantId(), cf.getEntityId()) + EntityId cfEntityId = cf.getEntityId(); + if (isProfileEntity(cfEntityId)) { + getOrFetchFromDBProfileEntities(cf.getTenantId(), cfEntityId) .forEach(entityId -> restoreState(cf, entityId)); } else { - restoreState(cf, cf.getEntityId()); + restoreState(cf, cfEntityId); } } } catch (Throwable t) { @@ -272,9 +275,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } case ASSET_PROFILE, DEVICE_PROFILE -> { log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); - fetchCommonArguments(calculatedFieldCtx, callback, commonArguments -> { + Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() + .filter(entry -> !isProfileEntity(entry.getValue().getEntityId())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + fetchArguments(tenantId, entityId, commonArguments, commonArgs -> { getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(targetEntityId -> { - initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArguments, callback); + initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArgs, callback); }); }); } @@ -473,7 +479,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (calculatedFieldState != null) { msgBuilder.setState(JacksonUtil.writeValueAsString(calculatedFieldState)); } - clusterService.pushMsgToCore(tenantId, entityId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); + clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); } private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { @@ -556,35 +562,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } - private void fetchCommonArguments(CalculatedFieldCtx calculatedFieldCtx, TbCallback callback, Consumer> onComplete) { - Map argumentValues = new HashMap<>(); - List> futures = new ArrayList<>(); - - calculatedFieldCtx.getArguments().forEach((key, argument) -> { - if (!EntityType.DEVICE_PROFILE.equals(argument.getEntityId().getEntityType()) && - !EntityType.ASSET_PROFILE.equals(argument.getEntityId().getEntityType())) { - futures.add(Futures.transform(fetchKvEntry(calculatedFieldCtx.getTenantId(), argument.getEntityId(), argument), - result -> { - argumentValues.put(key, result); - return result; - }, calculatedFieldCallbackExecutor)); - } - }); - - Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { - @Override - public void onSuccess(List results) { - onComplete.accept(argumentValues); - } - - @Override - public void onFailure(Throwable t) { - log.error("Failed to fetch common arguments", t); - callback.onFailure(t); - } - }, calculatedFieldCallbackExecutor); - } - private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { initializeStateForEntity(calculatedFieldCtx, entityId, new HashMap<>(), callback); } @@ -595,7 +572,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldCtx.getArguments().forEach((key, argument) -> { if (!commonArguments.containsKey(key)) { - futures.add(Futures.transform(fetchArgumentValue(calculatedFieldCtx, entityId, argument), + futures.add(Futures.transform(fetchArgumentValue(calculatedFieldCtx.getTenantId(), entityId, argument), result -> { argumentValues.put(key, result); return result; @@ -618,10 +595,25 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } - private ListenableFuture fetchArgumentValue(CalculatedFieldCtx calculatedFieldCtx, EntityId targetEntityId, Argument argument) { - TenantId tenantId = calculatedFieldCtx.getTenantId(); + private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { + Map argumentValues = new HashMap<>(); + List> futures = new ArrayList<>(); + necessaryArguments.forEach((key, argument) -> { + futures.add(Futures.transform(fetchArgumentValue(tenantId, entityId, argument), + result -> { + argumentValues.put(key, result); + return result; + }, calculatedFieldCallbackExecutor)); + }); + return Futures.transform(Futures.allAsList(futures), results -> { + onComplete.accept(argumentValues); + return null; + }, calculatedFieldCallbackExecutor); + } + + private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { EntityId argumentEntityId = argument.getEntityId(); - EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) + EntityId entityId = isProfileEntity(argumentEntityId) ? targetEntityId : argumentEntityId; return fetchKvEntry(tenantId, entityId, argument); @@ -654,11 +646,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); - return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? ArgumentEntry.createTsRollingArgument(Collections.emptyList()) : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); + return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); } private ListenableFuture transformSingleValueArgument(ListenableFuture> kvEntryFuture) { - return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldCallbackExecutor); + return Futures.transform(kvEntryFuture, kvEntry -> { + if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { + return ArgumentEntry.createSingleValueArgument(kvEntry.get()); + } else { + return SingleValueArgumentEntry.EMPTY; + } + }, calculatedFieldCallbackExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { @@ -676,48 +674,67 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues) { CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, this::fetchCalculatedFieldEntityState); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); + + Predicate> allArgsPresent = (args) -> + args.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && + !args.containsValue(SingleValueArgumentEntry.EMPTY) && !args.containsValue(TsRollingArgumentEntry.EMPTY); + + Consumer performUpdateState = (state) -> { + if (state.updateState(argumentValues)) { + calculatedFieldEntityCtx.setState(state); + TenantId tenantId = calculatedFieldCtx.getTenantId(); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); + if (tpi.isMyPartition()) { + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + } else { + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, state); + } + + if (allArgsPresent.test(state.getArguments())) { + performCalculation(calculatedFieldCtx, state, entityId); + } + } + }; CalculatedFieldState state = calculatedFieldEntityCtx.getState(); + boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); + if (!allKeysPresent) { - if (state == null) { - state = createStateByType(calculatedFieldCtx.getCfType()); + Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() + .filter(entry -> !argumentValues.containsKey(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) + .addListener(() -> performUpdateState.accept(state), + calculatedFieldCallbackExecutor); + return; } - if (state.updateState(argumentValues)) { - calculatedFieldEntityCtx.setState(state); - TenantId tenantId = calculatedFieldCtx.getTenantId(); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldCtx.getCfId()); - if (tpi.isMyPartition()) { - states.put(entityCtxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, state); - } + performUpdateState.accept(state); + } - boolean allArgsPresent = calculatedFieldCtx.getArguments().keySet().containsAll(state.getArguments().keySet()); - if (allArgsPresent) { - ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); - Futures.addCallback(resultFuture, new FutureCallback<>() { - @Override - public void onSuccess(CalculatedFieldResult result) { - if (result != null) { - pushMsgToRuleEngine(tenantId, entityId, result); - } - } + private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId) { + ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); + Futures.addCallback(resultFuture, new FutureCallback<>() { + @Override + public void onSuccess(CalculatedFieldResult result) { + if (result != null) { + pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); + } + } - @Override - public void onFailure(Throwable t) { - log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); - } - }, MoreExecutors.directExecutor()); + @Override + public void onFailure(Throwable t) { + log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); } - } + }, MoreExecutors.directExecutor()); } - private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId) { + private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); if (stateStr == null) { - return new CalculatedFieldEntityCtx(entityCtxId, null); + return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); } return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); } @@ -725,8 +742,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { try { String type = calculatedFieldResult.getType(); - TbMsgType msgType = "ATTRIBUTE".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; - TbMsgMetaData md = "ATTRIBUTE".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; + TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; + TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(payload)); clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); @@ -749,4 +766,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }; } + private boolean isProfileEntity(EntityId entityId) { + return EntityType.DEVICE_PROFILE.equals(entityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(entityId.getEntityType()); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 81a57580db..3f4fd5bdce 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; @AllArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { + public static final ArgumentEntry EMPTY = new SingleValueArgumentEntry(0); + private long ts; private Object value; @@ -39,6 +41,14 @@ public class SingleValueArgumentEntry implements ArgumentEntry { this.value = entry.getValue(); } + /** + * Internal constructor to create immutable SingleValueArgumentEntry.EMPTY + * */ + private SingleValueArgumentEntry(int ignored) { + this.ts = System.currentTimeMillis(); + this.value = null; + } + @Override public ArgumentType getType() { return ArgumentType.SINGLE_VALUE; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 49aae15ac1..4ffa391550 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -29,6 +29,8 @@ import java.util.TreeMap; @Slf4j public class TsRollingArgumentEntry implements ArgumentEntry { + public static final ArgumentEntry EMPTY = new TsRollingArgumentEntry(0); + private static final int MAX_ROLLING_ARGUMENT_ENTRY_SIZE = 1000; private TreeMap tsRecords = new TreeMap<>(); @@ -37,6 +39,13 @@ public class TsRollingArgumentEntry implements ArgumentEntry { addAllTsRecords(tsRecords); } + /** + * Internal constructor to create immutable TsRollingArgumentEntry.EMPTY + */ + private TsRollingArgumentEntry(int ignored) { + this.tsRecords = new TreeMap<>(); + } + @Override public ArgumentType getType() { return ArgumentType.TS_ROLLING; @@ -50,7 +59,9 @@ public class TsRollingArgumentEntry implements ArgumentEntry { @Override public boolean hasUpdatedValue(ArgumentEntry entry) { - return !tsRecords.containsKey(((SingleValueArgumentEntry) entry).getTs()); + return entry instanceof SingleValueArgumentEntry ? + !tsRecords.containsKey(((SingleValueArgumentEntry) entry).getTs()) : + !tsRecords.keySet().containsAll(((TsRollingArgumentEntry) entry).getTsRecords().keySet()); } @Override From 2d65a6c457182e2f7412f083b1be7069c47d7f2d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 23 Dec 2024 12:20:58 +0200 Subject: [PATCH 06/14] removed wildcard imports usage --- .../server/controller/BaseController.java | 62 ++++++++++++++++++- .../entitiy/EntityStateSourcingListener.java | 9 ++- .../DefaultTelemetrySubscriptionService.java | 17 +++-- .../rule/engine/util/TenantIdLoader.java | 33 +++++++++- .../rule/engine/util/TenantIdLoaderTest.java | 27 +++++++- 5 files changed, 136 insertions(+), 12 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 2d990d708d..9a980d009d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -42,7 +42,27 @@ import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.*; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.DashboardInfo; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceInfo; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.EntityViewInfo; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.HomeDashboardInfo; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.OtaPackageInfo; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TbResourceInfo; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.TenantInfo; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmComment; import org.thingsboard.server.common.data.alarm.AlarmInfo; @@ -57,7 +77,37 @@ import org.thingsboard.server.common.data.edge.EdgeInfo; import org.thingsboard.server.common.data.exception.EntityVersionMismatchException; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.*; +import org.thingsboard.server.common.data.id.AlarmCommentId; +import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.DomainId; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.MobileAppBundleId; +import org.thingsboard.server.common.data.id.MobileAppId; +import org.thingsboard.server.common.data.id.NotificationTargetId; +import org.thingsboard.server.common.data.id.OAuth2ClientId; +import org.thingsboard.server.common.data.id.OtaPackageId; +import org.thingsboard.server.common.data.id.QueueId; +import org.thingsboard.server.common.data.id.RpcId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.id.TbResourceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.id.UUIDBased; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.id.WidgetTypeId; +import org.thingsboard.server.common.data.id.WidgetsBundleId; import org.thingsboard.server.common.data.mobile.app.MobileApp; import org.thingsboard.server.common.data.mobile.bundle.MobileAppBundle; import org.thingsboard.server.common.data.notification.targets.NotificationTarget; @@ -140,7 +190,13 @@ import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import java.io.IOException; import java.net.URI; import java.nio.charset.StandardCharsets; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Function; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 83ce48c24e..ab8246ccf5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -23,7 +23,14 @@ import org.springframework.stereotype.Component; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.*; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TbResourceInfo; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index bf66a9f868..2ddfd22db3 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -29,7 +29,11 @@ import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.*; +import org.thingsboard.rule.engine.api.AttributesDeleteRequest; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; +import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; +import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; @@ -49,7 +53,13 @@ import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; -import java.util.*; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Consumer; @@ -231,8 +241,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer .onlyLatest(true) .callback(new FutureCallback<>() { @Override - public void onSuccess(@Nullable Void tmp) { - } + public void onSuccess(@Nullable Void tmp) {} @Override public void onFailure(Throwable t) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java index 4311d912e4..445205e88e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java @@ -19,7 +19,38 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.id.*; +import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.id.ApiUsageStateId; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.DomainId; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.MobileAppBundleId; +import org.thingsboard.server.common.data.id.MobileAppId; +import org.thingsboard.server.common.data.id.NotificationRequestId; +import org.thingsboard.server.common.data.id.NotificationRuleId; +import org.thingsboard.server.common.data.id.NotificationTargetId; +import org.thingsboard.server.common.data.id.NotificationTemplateId; +import org.thingsboard.server.common.data.id.OAuth2ClientId; +import org.thingsboard.server.common.data.id.OtaPackageId; +import org.thingsboard.server.common.data.id.QueueId; +import org.thingsboard.server.common.data.id.QueueStatsId; +import org.thingsboard.server.common.data.id.RpcId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.id.TbResourceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.id.WidgetTypeId; +import org.thingsboard.server.common.data.id.WidgetsBundleId; import org.thingsboard.server.common.data.rule.RuleNode; import java.util.UUID; diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java index a879c0e3a2..921b9e26d7 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java @@ -23,8 +23,23 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.AbstractListeningExecutor; -import org.thingsboard.rule.engine.api.*; -import org.thingsboard.server.common.data.*; +import org.thingsboard.rule.engine.api.RuleEngineAlarmService; +import org.thingsboard.rule.engine.api.RuleEngineApiUsageStateService; +import org.thingsboard.rule.engine.api.RuleEngineAssetProfileCache; +import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache; +import org.thingsboard.rule.engine.api.RuleEngineRpcService; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; @@ -32,7 +47,13 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.domain.Domain; import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.id.*; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.NotificationId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.mobile.app.MobileApp; import org.thingsboard.server.common.data.mobile.bundle.MobileAppBundle; import org.thingsboard.server.common.data.notification.NotificationRequest; From 1482d1c4bb9983f18ab9d6abb5405f1a68496ec9 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 23 Dec 2024 17:00:10 +0200 Subject: [PATCH 07/14] added logic to avoid looping --- .../cf/CalculatedFieldExecutionService.java | 3 +- ...efaultCalculatedFieldExecutionService.java | 57 +++++++------------ .../DefaultTelemetrySubscriptionService.java | 4 +- .../thingsboard/server/common/msg/TbMsg.java | 30 ++++++++-- common/message/src/main/proto/tbmsg.proto | 7 +++ .../engine/api/AttributesSaveRequest.java | 10 +++- .../engine/api/TimeseriesSaveRequest.java | 10 +++- .../engine/telemetry/TbMsgTimeseriesNode.java | 1 + 8 files changed, 76 insertions(+), 46 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 966b5d7277..ee9b6d115c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; @@ -27,7 +28,7 @@ public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List telemetry); + void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List calculatedFieldIds, List telemetry); void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 1128a75008..e5c119bba6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -18,11 +18,7 @@ package org.thingsboard.server.service.cf; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Lists; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.*; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Getter; @@ -41,25 +37,8 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; -import org.thingsboard.server.common.data.kv.BooleanDataEntry; -import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.kv.ReadTsKvQuery; -import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.id.*; +import org.thingsboard.server.common.data.kv.*; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; @@ -301,7 +280,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List telemetry) { + public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List calculatedFieldIds, List telemetry) { try { EntityType entityType = entityId.getEntityType(); if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType) || EntityType.CUSTOMER.equals(entityType) || EntityType.TENANT.equals(entityType)) { @@ -326,7 +305,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas )); if (!updatedTelemetry.isEmpty()) { - executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, updatedTelemetry); + executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, calculatedFieldIds, updatedTelemetry); } }); } @@ -335,7 +314,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List calculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); @@ -347,12 +326,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); if (isCommonEntity) { - getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); + getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, calculatedFieldIds)); } else { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, calculatedFieldIds); } } - default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues); + default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues, calculatedFieldIds); } log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); } @@ -583,7 +562,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { @Override public void onSuccess(List results) { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, Collections.emptyList()); callback.onSuccess(); } @@ -671,7 +650,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return new StringDataEntry(key, defaultValue); } - private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues) { + private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List calculatedFieldIds) { CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); @@ -693,7 +672,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } if (allArgsPresent.test(state.getArguments())) { - performCalculation(calculatedFieldCtx, state, entityId); + performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); } } }; @@ -714,13 +693,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas performUpdateState.accept(state); } - private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId) { + private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId, List calculatedFieldIds) { ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); Futures.addCallback(resultFuture, new FutureCallback<>() { @Override public void onSuccess(CalculatedFieldResult result) { if (result != null) { - pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); + pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), calculatedFieldCtx.getCfId(), entityId, result, calculatedFieldIds); } } @@ -739,13 +718,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); } - private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { + private void pushMsgToRuleEngine(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult, List calculatedFieldIds) { try { String type = calculatedFieldResult.getType(); TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); - TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(payload)); + if (calculatedFieldIds.contains(calculatedFieldId)) { + throw new IllegalArgumentException("Calculated field [" + calculatedFieldId.getId() + "] refers to itself, causing an infinite loop."); + } + calculatedFieldIds.add(calculatedFieldId); + TbMsg msg = TbMsg.newMsg().type(msgType).originator(originatorId).calculatedFieldIds(calculatedFieldIds).metaData(md).data(JacksonUtil.writeValueAsString(payload)).build(); clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); } catch (Exception e) { log.warn("[{}] Failed to push message to rule engine. CalculatedFieldResult: {}", originatorId, calculatedFieldResult, e); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 2ddfd22db3..e35d3cead6 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -152,7 +152,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } - calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, request.getEntries()); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, request.getCalculatedFieldIds(), request.getEntries()); return saveFuture; } @@ -168,7 +168,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); - calculatedFieldExecutionService.onTelemetryUpdate(request.getTenantId(), request.getEntityId(), request.getEntries()); + calculatedFieldExecutionService.onTelemetryUpdate(request.getTenantId(), request.getEntityId(), request.getCalculatedFieldIds(), request.getEntries()); } @Override diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 64e05770fe..993fa24ff3 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -34,6 +34,8 @@ import org.thingsboard.server.common.msg.gen.MsgProtos; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; import java.util.UUID; @@ -64,6 +66,8 @@ public final class TbMsg implements Serializable { private final UUID correlationId; private final Integer partition; + private final List calculatedFieldIds; + @Getter(value = AccessLevel.NONE) @JsonIgnore //This field is not serialized because we use queues and there is no need to do it @@ -112,7 +116,7 @@ public final class TbMsg implements Serializable { } private TbMsg(String queueName, UUID id, long ts, TbMsgType internalType, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, UUID correlationId, Integer partition, TbMsgProcessingCtx ctx, TbMsgCallback callback) { + RuleChainId ruleChainId, RuleNodeId ruleNodeId, UUID correlationId, Integer partition, List calculatedFieldIds, TbMsgProcessingCtx ctx, TbMsgCallback callback) { this.id = id != null ? id : UUID.randomUUID(); this.queueName = queueName; if (ts > 0) { @@ -139,6 +143,7 @@ public final class TbMsg implements Serializable { this.ruleNodeId = ruleNodeId; this.correlationId = correlationId; this.partition = partition; + this.calculatedFieldIds = calculatedFieldIds; this.ctx = ctx != null ? ctx : new TbMsgProcessingCtx(); this.callback = Objects.requireNonNullElse(callback, TbMsgCallback.EMPTY); } @@ -200,6 +205,7 @@ public final class TbMsg implements Serializable { RuleNodeId ruleNodeId = null; UUID correlationId = null; Integer partition = null; + List calculatedFieldIds = new ArrayList<>(); if (proto.getCustomerIdMSB() != 0L && proto.getCustomerIdLSB() != 0L) { customerId = new CustomerId(new UUID(proto.getCustomerIdMSB(), proto.getCustomerIdLSB())); } @@ -214,6 +220,14 @@ public final class TbMsg implements Serializable { partition = proto.getPartition(); } + for (MsgProtos.CalculatedFieldIdProto cfIdProto : proto.getCalculatedFieldsList()) { + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID( + cfIdProto.getCalculatedFieldIdMSB(), + cfIdProto.getCalculatedFieldIdLSB() + )); + calculatedFieldIds.add(calculatedFieldId); + } + TbMsgProcessingCtx ctx; if (proto.hasCtx()) { ctx = TbMsgProcessingCtx.fromProto(proto.getCtx()); @@ -224,7 +238,7 @@ public final class TbMsg implements Serializable { TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), null, proto.getType(), entityId, customerId, - metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, correlationId, partition, ctx, callback); + metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, correlationId, partition, calculatedFieldIds, ctx, callback); } catch (InvalidProtocolBufferException e) { throw new IllegalStateException("Could not parse protobuf for TbMsg", e); } @@ -343,10 +357,12 @@ public final class TbMsg implements Serializable { protected RuleNodeId ruleNodeId; protected UUID correlationId; protected Integer partition; + protected List calculatedFieldIds; protected TbMsgProcessingCtx ctx; protected TbMsgCallback callback; - TbMsgBuilder() {} + TbMsgBuilder() { + } TbMsgBuilder(TbMsg tbMsg) { this.queueName = tbMsg.queueName; @@ -363,6 +379,7 @@ public final class TbMsg implements Serializable { this.ruleNodeId = tbMsg.ruleNodeId; this.correlationId = tbMsg.correlationId; this.partition = tbMsg.partition; + this.calculatedFieldIds = tbMsg.calculatedFieldIds; this.ctx = tbMsg.ctx; this.callback = tbMsg.callback; } @@ -454,6 +471,11 @@ public final class TbMsg implements Serializable { return this; } + public TbMsgBuilder calculatedFieldIds(List calculatedFieldIds) { + this.calculatedFieldIds = calculatedFieldIds; + return this; + } + public TbMsgBuilder ctx(TbMsgProcessingCtx ctx) { this.ctx = ctx; return this; @@ -465,7 +487,7 @@ public final class TbMsg implements Serializable { } public TbMsg build() { - return new TbMsg(queueName, id, ts, internalType, type, originator, customerId, metaData, dataType, data, ruleChainId, ruleNodeId, correlationId, partition, ctx, callback); + return new TbMsg(queueName, id, ts, internalType, type, originator, customerId, metaData, dataType, data, ruleChainId, ruleNodeId, correlationId, partition, calculatedFieldIds, ctx, callback); } public String toString() { diff --git a/common/message/src/main/proto/tbmsg.proto b/common/message/src/main/proto/tbmsg.proto index fc9265aa14..36ab09f8d4 100644 --- a/common/message/src/main/proto/tbmsg.proto +++ b/common/message/src/main/proto/tbmsg.proto @@ -70,4 +70,11 @@ message TbMsgProto { int64 correlationIdMSB = 20; int64 correlationIdLSB = 21; int32 partition = 22; + + repeated CalculatedFieldIdProto calculatedFields = 23; +} + +message CalculatedFieldIdProto { + int64 calculatedFieldIdMSB = 1; + int64 calculatedFieldIdLSB = 2; } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java index 22fa8de6de..9747a6033e 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java @@ -22,6 +22,7 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.ToString; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -40,6 +41,7 @@ public class AttributesSaveRequest { private final AttributeScope scope; private final List entries; private final boolean notifyDevice; + private final List calculatedFieldIds; private final FutureCallback callback; public static Builder builder() { @@ -53,6 +55,7 @@ public class AttributesSaveRequest { private AttributeScope scope; private List entries; private boolean notifyDevice = true; + private List calculatedFieldIds; private FutureCallback callback; Builder() {} @@ -100,6 +103,11 @@ public class AttributesSaveRequest { return this; } + public Builder calculatedFieldIds(List calculatedFieldIds) { + this.calculatedFieldIds = calculatedFieldIds; + return this; + } + public Builder callback(FutureCallback callback) { this.callback = callback; return this; @@ -120,7 +128,7 @@ public class AttributesSaveRequest { } public AttributesSaveRequest build() { - return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, callback); + return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, calculatedFieldIds, callback); } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java index 2b5881212d..12afa2d939 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.SettableFuture; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -40,6 +41,7 @@ public class TimeseriesSaveRequest { private final long ttl; private final boolean saveLatest; private final boolean onlyLatest; + private final List calculatedFieldIds; private final FutureCallback callback; public static Builder builder() { @@ -56,6 +58,7 @@ public class TimeseriesSaveRequest { private FutureCallback callback; private boolean saveLatest = true; private boolean onlyLatest; + private List calculatedFieldIds; Builder() {} @@ -103,6 +106,11 @@ public class TimeseriesSaveRequest { return this; } + public Builder calculatedFieldIds(List calculatedFieldIds) { + this.calculatedFieldIds = calculatedFieldIds; + return this; + } + public Builder callback(FutureCallback callback) { this.callback = callback; return this; @@ -123,7 +131,7 @@ public class TimeseriesSaveRequest { } public TimeseriesSaveRequest build() { - return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, callback); + return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, calculatedFieldIds, callback); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 27f45feb47..386e56320a 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -112,6 +112,7 @@ public class TbMsgTimeseriesNode implements TbNode { .entries(tsKvEntryList) .ttl(ttl) .saveLatest(!config.isSkipLatestPersistence()) + .calculatedFieldIds(msg.getCalculatedFieldIds()) .callback(new TelemetryNodeCallback(ctx, msg)) .build()); } From a9f39e4917d0da31bdc89a45a8c5df344eec9d1f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 24 Dec 2024 12:50:27 +0200 Subject: [PATCH 08/14] added cache class --- .../service/cf/CalculatedFieldCache.java | 43 +++ .../cf/DefaultCalculatedFieldCache.java | 212 +++++++++++++ ...efaultCalculatedFieldExecutionService.java | 279 ++++++++++-------- .../thingsboard/server/common/msg/TbMsg.java | 15 +- common/proto/src/main/proto/queue.proto | 29 +- .../engine/telemetry/TbMsgAttributesNode.java | 1 + 6 files changed, 454 insertions(+), 125 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java new file mode 100644 index 0000000000..f953e57cc3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.List; +import java.util.Set; + +public interface CalculatedFieldCache { + + CalculatedField getCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + + CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); + + Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); + + void evict(CalculatedFieldId calculatedFieldId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java new file mode 100644 index 0000000000..1fc2a90e07 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -0,0 +1,212 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import jakarta.annotation.PostConstruct; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultCalculatedFieldCache implements CalculatedFieldCache { + + private final Lock calculatedFieldFetchLock = new ReentrantLock(); + + private final CalculatedFieldService calculatedFieldService; + private final AssetService assetService; + private final DeviceService deviceService; + + private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); + private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); + + @Value("${calculatedField.initFetchPackSize:50000}") + @Getter + private int initFetchPackSize; + + + @PostConstruct + public void init() { + // to discuss: fetch on start or fetch on demand + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); + PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); + cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + } + + @Override + public CalculatedField getCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + CalculatedField calculatedField = calculatedFields.get(calculatedFieldId); + if (calculatedField == null) { + calculatedFieldFetchLock.lock(); + try { + calculatedField = calculatedFields.get(calculatedFieldId); + if (calculatedField == null) { + calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); + if (calculatedField != null) { + calculatedFields.put(calculatedFieldId, calculatedField); + log.debug("[{}] Fetch calculated field into cache: {}", calculatedFieldId, calculatedField); + } + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found calculated field in cache: {}", calculatedFieldId, calculatedField); + return calculatedField; + } + + @Override + public List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + List cfLinks = calculatedFieldLinks.get(calculatedFieldId); + if (cfLinks == null) { + calculatedFieldFetchLock.lock(); + try { + cfLinks = calculatedFieldLinks.get(calculatedFieldId); + if (cfLinks == null) { + cfLinks = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); + if (cfLinks != null) { + calculatedFieldLinks.put(calculatedFieldId, cfLinks); + log.debug("[{}] Fetch calculated field links into cache: {}", calculatedFieldId, cfLinks); + } + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found calculated field links in cache: {}", calculatedFieldId, cfLinks); + return cfLinks; + } + + @Override + public List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { + List cfLinks = entityIdCalculatedFieldLinks.get(entityId); + if (cfLinks == null) { + calculatedFieldFetchLock.lock(); + try { + cfLinks = entityIdCalculatedFieldLinks.get(entityId); + if (cfLinks == null) { + cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); + if (cfLinks != null) { + entityIdCalculatedFieldLinks.put(entityId, cfLinks); + log.debug("[{}] Fetch calculated field links by entity id into cache: {}", entityId, cfLinks); + } + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found calculated field links by entity id in cache: {}", entityId, cfLinks); + return cfLinks; + } + + @Override + public CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService) { + CalculatedFieldCtx ctx = calculatedFieldsCtx.get(calculatedFieldId); + if (ctx == null) { + calculatedFieldFetchLock.lock(); + try { + ctx = calculatedFieldsCtx.get(calculatedFieldId); + if (ctx == null) { + CalculatedField calculatedField = getCalculatedField(tenantId, calculatedFieldId); + if (calculatedField != null) { + ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService); + calculatedFieldsCtx.put(calculatedFieldId, ctx); + log.debug("[{}] Put calculated field ctx into cache: {}", calculatedFieldId, ctx); + } + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found calculated field ctx in cache: {}", calculatedFieldId, ctx); + return ctx; + } + + @Override + public Set getEntitiesByProfile(TenantId tenantId, EntityId entityProfileId) { + Set entities = profileEntities.get(entityProfileId); + if (entities == null) { + calculatedFieldFetchLock.lock(); + try { + entities = profileEntities.get(entityProfileId); + if (entities == null) { + entities = switch (entityProfileId.getEntityType()) { + case ASSET_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { + Set assetIds = new HashSet<>(); + (new PageDataIterable<>(pageLink -> + assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) profileId, pageLink), initFetchPackSize)).forEach(assetIds::add); + return assetIds; + }); + case DEVICE_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { + Set deviceIds = new HashSet<>(); + (new PageDataIterable<>(pageLink -> + deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityProfileId, pageLink), initFetchPackSize)).forEach(deviceIds::add); + return deviceIds; + }); + default -> + throw new IllegalArgumentException("Entity type should be ASSET_PROFILE or DEVICE_PROFILE."); + }; + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found entities by profile in cache: {}", entityProfileId, entities); + return entities; + } + + @Override + public void evict(CalculatedFieldId calculatedFieldId) { + CalculatedField oldCalculatedField = calculatedFields.remove(calculatedFieldId); + log.debug("[{}] evict calculated field from cache: {}", calculatedFieldId, oldCalculatedField); + calculatedFieldLinks.remove(calculatedFieldId); + log.debug("[{}] evict calculated field links from cache: {}", calculatedFieldId, oldCalculatedField); + calculatedFieldsCtx.remove(calculatedFieldId); + log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); + entityIdCalculatedFieldLinks.forEach((entityId, calculatedFieldLinks) -> calculatedFieldLinks.removeIf(link -> link.getCalculatedFieldId().equals(calculatedFieldId))); + log.debug("[{}] evict calculated field links from cached links by entity id: {}", calculatedFieldId, oldCalculatedField); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index e5c119bba6..ff78d2925a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -18,7 +18,11 @@ package org.thingsboard.server.service.cf; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Lists; -import com.google.common.util.concurrent.*; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Getter; @@ -37,8 +41,23 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.*; -import org.thingsboard.server.common.data.kv.*; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; @@ -46,10 +65,8 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; -import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -67,8 +84,8 @@ import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -77,7 +94,6 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; -import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -90,10 +106,9 @@ import static org.thingsboard.server.common.data.DataConstants.SCOPE; public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBasedService implements CalculatedFieldExecutionService { private final CalculatedFieldService calculatedFieldService; - private final AssetService assetService; - private final DeviceService deviceService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; + private final CalculatedFieldCache calculatedFieldCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; private final RocksDBService rocksDBService; @@ -103,13 +118,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; - private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); - private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); - private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); - private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); - private static final int MAX_LAST_RECORDS_VALUE = 1024; @Value("${calculatedField.initFetchPackSize:50000}") @@ -123,20 +133,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.submit(this::fetchCalculatedFields); - } - - private void fetchCalculatedFields() { - PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); - cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); - PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); - cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class))); - states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); } @PreDestroy public void stop() { + super.stop(); if (calculatedFieldExecutor != null) { calculatedFieldExecutor.shutdownNow(); } @@ -183,7 +184,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas for (CalculatedField cf : partition) { EntityId cfEntityId = cf.getEntityId(); if (isProfileEntity(cfEntityId)) { - getOrFetchFromDBProfileEntities(cf.getTenantId(), cfEntityId) + calculatedFieldCache.getEntitiesByProfile(cf.getTenantId(), cfEntityId) .forEach(entityId -> restoreState(cf, entityId)); } else { restoreState(cf, cfEntityId); @@ -206,7 +207,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (storedState != null) { CalculatedFieldEntityCtx restoredCtx = JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class); - calculatedFieldsCtx.putIfAbsent(cf.getId(), new CalculatedFieldCtx(cf, tbelInvokeService)); states.put(ctxId, restoredCtx); log.info("Restored state for CalculatedField [{}]", cf.getId()); } else { @@ -220,7 +220,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void cleanupEntity(CalculatedFieldId calculatedFieldId) { - calculatedFields.remove(calculatedFieldId); states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); } @@ -235,7 +234,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas onCalculatedFieldDelete(tenantId, calculatedFieldId, callback); callback.onSuccess(); } - CalculatedField cf = getOrFetchFromDb(tenantId, calculatedFieldId); + CalculatedField cf = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); @@ -245,8 +244,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } if (cf != null) { EntityId entityId = cf.getEntityId(); - CalculatedFieldCtx calculatedFieldCtx = new CalculatedFieldCtx(cf, tbelInvokeService); - calculatedFieldsCtx.put(calculatedFieldId, calculatedFieldCtx); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); switch (entityId.getEntityType()) { case ASSET, DEVICE -> { log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); @@ -258,7 +256,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .filter(entry -> !isProfileEntity(entry.getValue().getEntityId())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(tenantId, entityId, commonArguments, commonArgs -> { - getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(targetEntityId -> { + calculatedFieldCache.getEntitiesByProfile(tenantId, entityId).forEach(targetEntityId -> { initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArgs, callback); }); }); @@ -290,8 +288,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } else if (EntityType.DEVICE.equals(entityType)) { profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); } - List cfLinks = new ArrayList<>(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); - Optional.ofNullable(profileId).ifPresent(id -> cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, id))); + List cfLinks = calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId); + Optional.ofNullable(profileId).ifPresent(id -> calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, id)); cfLinks.forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); Map attributes = link.getConfiguration().getAttributes(); @@ -316,8 +314,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List calculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); - CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); + CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); Map argumentValues = updatedTelemetry.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); @@ -326,7 +324,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); if (isCommonEntity) { - getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, calculatedFieldIds)); + calculatedFieldCache.getEntitiesByProfile(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, calculatedFieldIds)); } else { updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, calculatedFieldIds); } @@ -353,28 +351,61 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return entry.getKey(); } + private Object deserializeObjectProto(TransportProtos.ObjectProto objectProto) { + try { + String type = objectProto.getType(); + String value = objectProto.getValue(); + return switch (type) { + case "java.lang.String" -> value; + case "java.lang.Integer" -> Integer.parseInt(value); + case "java.lang.Long" -> Long.parseLong(value); + case "java.lang.Double" -> Double.parseDouble(value); + case "java.lang.Boolean" -> Boolean.parseBoolean(value); + default -> throw new IllegalArgumentException("Unsupported object type: " + type); + }; + } catch (Exception e) { + log.error("Failed to deserialize ObjectProto: [{}]", objectProto, e); + return null; + } + } + @Override public void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - String state = proto.getState(); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = state.isEmpty() ? JacksonUtil.fromString(state, CalculatedFieldEntityCtx.class) : null; + List calculatedFieldIds = new ArrayList<>(); + for (TransportProtos.CalculatedFieldIdProto cfIdProto : proto.getCalculatedFieldsList()) { + CalculatedFieldId cfId = new CalculatedFieldId(new UUID( + cfIdProto.getCalculatedFieldIdMSB(), + cfIdProto.getCalculatedFieldIdLSB() + )); + calculatedFieldIds.add(cfId); + } - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); - if (tpi.isMyPartition()) { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId()); - if (calculatedFieldEntityCtx != null) { - states.put(ctxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), state); + Map argumentsMap = new HashMap<>(); + proto.getArgumentsMap().forEach((key, entryProto) -> { + ArgumentEntry argumentEntry; + if (entryProto.hasTsRecords()) { + TsRollingArgumentEntry tsRollingArgumentEntry = new TsRollingArgumentEntry(); + entryProto.getTsRecords().getTsRecordsMap().forEach((ts, objectProto) -> { + Object value = deserializeObjectProto(objectProto); + tsRollingArgumentEntry.getTsRecords().put(ts, value); + }); + argumentEntry = tsRollingArgumentEntry; + } else if (entryProto.hasSingleValue()) { + TransportProtos.SingleValueProto singleRecordProto = entryProto.getSingleValue(); + Object value = deserializeObjectProto(singleRecordProto.getValue()); + argumentEntry = new SingleValueArgumentEntry(singleRecordProto.getTs(), value); } else { - states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + throw new IllegalArgumentException("Unsupported ArgumentEntryProto type"); } - } else { - log.debug("[{}] Calculated Field belongs to external partition {}", calculatedFieldId, tpi.getFullTopicName()); - } + argumentsMap.put(key, argumentEntry); + }); + + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, calculatedFieldIds); } catch (Exception e) { log.trace("Failed to process calculated field update state msg: [{}]", proto, e); } @@ -389,8 +420,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - profileEntities.get(oldProfileId).remove(entityId); - profileEntities.computeIfAbsent(newProfileId, id -> new HashSet<>()).add(entityId); + calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfileId).remove(entityId); + calculatedFieldCache.getEntitiesByProfile(tenantId, newProfileId).add(entityId); calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> { @@ -400,7 +431,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.remove(ctxId); rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, null); + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, Collections.emptyList(), null); } }); @@ -419,12 +450,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); if (proto.getDeleted()) { log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - profileEntities.get(profileId).remove(entityId); + calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).remove(entityId); List calculatedFieldIds = Stream.concat( - calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId).stream() - .map(CalculatedFieldLink::getCalculatedFieldId), - calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, profileId).stream() - .map(CalculatedFieldLink::getCalculatedFieldId) + calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream().map(CalculatedFieldLink::getCalculatedFieldId), + calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream().map(CalculatedFieldLink::getCalculatedFieldId) ).toList(); calculatedFieldIds.forEach(cfId -> { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); @@ -433,12 +462,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.remove(ctxId); rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, null); + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, Collections.emptyList(), null); } }); } else { log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - profileEntities.computeIfAbsent(profileId, id -> new HashSet<>()).add(entityId); + calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).add(entityId); initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); } } catch (Exception e) { @@ -446,7 +475,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, CalculatedFieldState calculatedFieldState) { + private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List calculatedFieldIds, Map argumentValues) { TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = TransportProtos.CalculatedFieldStateMsgProto.newBuilder() .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) @@ -455,20 +484,45 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .setEntityType(entityId.getEntityType().name()) .setEntityIdMSB(entityId.getId().getMostSignificantBits()) .setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - if (calculatedFieldState != null) { - msgBuilder.setState(JacksonUtil.writeValueAsString(calculatedFieldState)); + + if (argumentValues != null) { + argumentValues.forEach((key, argumentEntry) -> { + TransportProtos.ArgumentEntryProto.Builder argumentEntryProtoBuilder = TransportProtos.ArgumentEntryProto.newBuilder(); + + if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { + TransportProtos.TsRollingProto.Builder tsRollingProtoBuilder = TransportProtos.TsRollingProto.newBuilder(); + + tsRollingArgumentEntry.getTsRecords().forEach((ts, value) -> { + TransportProtos.ObjectProto.Builder objectProtoBuilder = TransportProtos.ObjectProto.newBuilder() + .setType(value.getClass().getName()) + .setValue(value.toString()); + tsRollingProtoBuilder.putTsRecords(ts, objectProtoBuilder.build()); + }); + + argumentEntryProtoBuilder.setTsRecords(tsRollingProtoBuilder.build()); + } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + TransportProtos.SingleValueProto.Builder singleRecordProtoBuilder = TransportProtos.SingleValueProto.newBuilder() + .setTs(singleValueArgumentEntry.getTs()) + .setValue(TransportProtos.ObjectProto.newBuilder() + .setType(singleValueArgumentEntry.getValue().getClass().getName()) + .setValue(singleValueArgumentEntry.getValue().toString()) + .build()); + argumentEntryProtoBuilder.setSingleValue(singleRecordProtoBuilder.build()); + } + + msgBuilder.putArguments(key, argumentEntryProtoBuilder.build()); + }); } + clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); } private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = getOrFetchFromDb(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); + CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); boolean shouldReinit = true; if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { onCalculatedFieldDelete(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId(), callback); } else { - calculatedFields.put(updatedCalculatedField.getId(), updatedCalculatedField); - calculatedFieldsCtx.put(updatedCalculatedField.getId(), new CalculatedFieldCtx(updatedCalculatedField, tbelInvokeService)); callback.onSuccess(); shouldReinit = false; } @@ -483,8 +537,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (calculatedFieldIds != null) { calculatedFieldIds.remove(calculatedFieldId); } - calculatedFields.remove(calculatedFieldId); - calculatedFieldsCtx.remove(calculatedFieldId); + calculatedFieldCache.evict(calculatedFieldId); states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); List statesToRemove = states.keySet().stream() .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) @@ -497,28 +550,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private CalculatedField getOrFetchFromDb(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - return calculatedFields.computeIfAbsent(calculatedFieldId, cfId -> calculatedFieldService.findById(tenantId, calculatedFieldId)); - } - - private Set getOrFetchFromDBProfileEntities(TenantId tenantId, EntityId entityProfileId) { - return switch (entityProfileId.getEntityType()) { - case ASSET_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { - Set assetIds = new HashSet<>(); - (new PageDataIterable<>(pageLink -> - assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) profileId, pageLink), initFetchPackSize)).forEach(assetIds::add); - return assetIds; - }); - case DEVICE_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { - Set deviceIds = new HashSet<>(); - (new PageDataIterable<>(pageLink -> - deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityProfileId, pageLink), initFetchPackSize)).forEach(deviceIds::add); - return deviceIds; - }); - default -> throw new IllegalArgumentException("Entity type should be ASSET_PROFILE or DEVICE_PROFILE."); - }; - } - private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { if (oldCalculatedField == null) { return true; @@ -537,7 +568,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) .stream() - .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) + .map(cfId -> calculatedFieldCache.getCalculatedFieldCtx(tenantId, cfId, tbelInvokeService)) .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } @@ -562,7 +593,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { @Override public void onSuccess(List results) { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, Collections.emptyList()); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, new ArrayList<>()); callback.onSuccess(); } @@ -651,46 +682,47 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List calculatedFieldIds) { + TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); - - Predicate> allArgsPresent = (args) -> - args.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && - !args.containsValue(SingleValueArgumentEntry.EMPTY) && !args.containsValue(TsRollingArgumentEntry.EMPTY); - - Consumer performUpdateState = (state) -> { - if (state.updateState(argumentValues)) { - calculatedFieldEntityCtx.setState(state); - TenantId tenantId = calculatedFieldCtx.getTenantId(); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); - if (tpi.isMyPartition()) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); + if (tpi.isMyPartition()) { + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); + + Consumer performUpdateState = (state) -> { + if (state.updateState(argumentValues)) { + calculatedFieldEntityCtx.setState(state); states.put(entityCtxId, calculatedFieldEntityCtx); rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, state); + Map arguments = state.getArguments(); + boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && + !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); + if (allArgsPresent) { + performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); + } } + }; - if (allArgsPresent.test(state.getArguments())) { - performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); - } - } - }; + CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); - if (!allKeysPresent) { + boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); + if (!allKeysPresent) { - Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentValues.containsKey(entry.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() + .filter(entry -> !argumentValues.containsKey(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) - .addListener(() -> performUpdateState.accept(state), - calculatedFieldCallbackExecutor); - return; + fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) + .addListener(() -> performUpdateState.accept(state), + calculatedFieldCallbackExecutor); + return; + } + performUpdateState.accept(state); + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + } else { + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentValues); } - performUpdateState.accept(state); } private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId, List calculatedFieldIds) { @@ -724,6 +756,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); + if (calculatedFieldIds == null) { + calculatedFieldIds = new ArrayList<>(); + } if (calculatedFieldIds.contains(calculatedFieldId)) { throw new IllegalArgumentException("Calculated field [" + calculatedFieldId.getId() + "] refers to itself, causing an infinite loop."); } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 993fa24ff3..0805175f77 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -24,6 +24,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -191,6 +192,16 @@ public final class TbMsg implements Serializable { builder.setPartition(msg.getPartition()); } + if (msg.getCalculatedFieldIds() != null) { + for (CalculatedFieldId calculatedFieldId : msg.getCalculatedFieldIds()) { + MsgProtos.CalculatedFieldIdProto calculatedFieldIdProto = MsgProtos.CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) + .build(); + builder.addCalculatedFields(calculatedFieldIdProto); + } + } + builder.setCtx(msg.ctx.toProto()); return builder.build().toByteArray(); } @@ -495,8 +506,8 @@ public final class TbMsg implements Serializable { ", type=" + this.type + ", internalType=" + this.internalType + ", originator=" + this.originator + ", customerId=" + this.customerId + ", metaData=" + this.metaData + ", dataType=" + this.dataType + ", data=" + this.data + ", ruleChainId=" + this.ruleChainId + ", ruleNodeId=" + this.ruleNodeId + - ", correlationId=" + this.correlationId + ", partition=" + this.partition + ", ctx=" + this.ctx + - ", callback=" + this.callback + ")"; + ", correlationId=" + this.correlationId + ", partition=" + this.partition + ", calculatedFields=" + this.calculatedFieldIds + + ", ctx=" + this.ctx + ", callback=" + this.callback + ")"; } } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 3df19e313e..f6f7afd5d8 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -817,7 +817,34 @@ message CalculatedFieldStateMsgProto { string entityType = 5; int64 entityIdMSB = 6; int64 entityIdLSB = 7; - string state = 8; + repeated CalculatedFieldIdProto calculatedFields = 8; + map arguments = 9; +} + +message CalculatedFieldIdProto { + int64 calculatedFieldIdMSB = 1; + int64 calculatedFieldIdLSB = 2; +} + +message ArgumentEntryProto { + oneof entry_type { + TsRollingProto tsRecords = 1; + SingleValueProto singleValue = 2; + } +} + +message TsRollingProto { + map tsRecords = 1; +} + +message SingleValueProto { + int64 ts = 1; + ObjectProto value = 2; +} + +message ObjectProto { + string type = 1; + string value = 2; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index e83a125265..20d0dda42f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -125,6 +125,7 @@ public class TbMsgAttributesNode implements TbNode { .scope(scope) .entries(attributes) .notifyDevice(config.isNotifyDevice() || checkNotifyDeviceMdValue(msg.getMetaData().getValue(NOTIFY_DEVICE_METADATA_KEY))) + .calculatedFieldIds(msg.getCalculatedFieldIds()) .callback(callback) .build()); } From 2dfbe2240d4bd22904e3d5edd6a7edeb15b0d1a5 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 24 Dec 2024 16:25:01 +0200 Subject: [PATCH 09/14] added implementation to handle profile events when not my partition --- ...efaultCalculatedFieldExecutionService.java | 510 +++++++++--------- .../server/common/util/ProtoUtils.java | 40 ++ common/proto/src/main/proto/queue.proto | 14 +- 3 files changed, 306 insertions(+), 258 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index ff78d2925a..6e7536825a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -84,7 +84,6 @@ import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -98,6 +97,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import static org.thingsboard.server.common.data.DataConstants.SCOPE; +import static org.thingsboard.server.common.util.ProtoUtils.fromObjectProto; +import static org.thingsboard.server.common.util.ProtoUtils.toObjectProto; @TbCoreComponent @Service @@ -229,6 +230,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + if (!tpi.isMyPartition()) { + clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldMsg(proto).build(), null); + log.debug("[{}][{}] Calculated field belongs to external partition. Probably rebalancing is in progress. Topic: {}", tenantId, calculatedFieldId, tpi.getFullTopicName()); + callback.onFailure(new RuntimeException("Calculated field belongs to external partition " + tpi.getFullTopicName() + "!")); + } if (proto.getDeleted()) { log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); onCalculatedFieldDelete(tenantId, calculatedFieldId, callback); @@ -277,6 +284,54 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { + CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); + boolean shouldReinit = true; + if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { + onCalculatedFieldDelete(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId(), callback); + } else { + callback.onSuccess(); + shouldReinit = false; + } + return shouldReinit; + } + + private void onCalculatedFieldDelete(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbCallback callback) { + try { + cleanupEntity(calculatedFieldId); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + Set calculatedFieldIds = partitionedEntities.get(tpi); + if (calculatedFieldIds != null) { + calculatedFieldIds.remove(calculatedFieldId); + } + calculatedFieldCache.evict(calculatedFieldId); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); + List statesToRemove = states.keySet().stream() + .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) + .map(JacksonUtil::writeValueAsString) + .toList(); + rocksDBService.deleteAll(statesToRemove); + } catch (Exception e) { + log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); + callback.onFailure(e); + } + } + + private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { + if (oldCalculatedField == null) { + return true; + } + boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); + boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); + CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); + CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); + boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); + boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); + boolean expressionChanged = !oldConfig.getExpression().equals(newConfig.getExpression()); + + return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; + } + @Override public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List calculatedFieldIds, List telemetry) { try { @@ -288,8 +343,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } else if (EntityType.DEVICE.equals(entityType)) { profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); } - List cfLinks = calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId); - Optional.ofNullable(profileId).ifPresent(id -> calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, id)); + List cfLinks = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId)); + Optional.ofNullable(profileId).ifPresent(id -> { + cfLinks.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, id)); + }); cfLinks.forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); Map attributes = link.getConfiguration().getAttributes(); @@ -312,6 +369,23 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { + if (entry instanceof AttributeKvEntry) { + return attributes.entrySet().stream() + .filter(attr -> attr.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } else if (entry instanceof TsKvEntry) { + return timeSeries.entrySet().stream() + .filter(ts -> ts.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } + return entry.getKey(); + } + private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List calculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); @@ -334,75 +408,23 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); } - private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { - if (entry instanceof AttributeKvEntry) { - return attributes.entrySet().stream() - .filter(attr -> attr.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } else if (entry instanceof TsKvEntry) { - return timeSeries.entrySet().stream() - .filter(ts -> ts.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } - return entry.getKey(); - } - - private Object deserializeObjectProto(TransportProtos.ObjectProto objectProto) { - try { - String type = objectProto.getType(); - String value = objectProto.getValue(); - return switch (type) { - case "java.lang.String" -> value; - case "java.lang.Integer" -> Integer.parseInt(value); - case "java.lang.Long" -> Long.parseLong(value); - case "java.lang.Double" -> Double.parseDouble(value); - case "java.lang.Boolean" -> Boolean.parseBoolean(value); - default -> throw new IllegalArgumentException("Unsupported object type: " + type); - }; - } catch (Exception e) { - log.error("Failed to deserialize ObjectProto: [{}]", objectProto, e); - return null; - } - } - @Override public void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - List calculatedFieldIds = new ArrayList<>(); - for (TransportProtos.CalculatedFieldIdProto cfIdProto : proto.getCalculatedFieldsList()) { - CalculatedFieldId cfId = new CalculatedFieldId(new UUID( - cfIdProto.getCalculatedFieldIdMSB(), - cfIdProto.getCalculatedFieldIdLSB() - )); - calculatedFieldIds.add(cfId); + + if (proto.getClear()) { + clearState(tenantId, calculatedFieldId, entityId); + return; } - Map argumentsMap = new HashMap<>(); - proto.getArgumentsMap().forEach((key, entryProto) -> { - ArgumentEntry argumentEntry; - if (entryProto.hasTsRecords()) { - TsRollingArgumentEntry tsRollingArgumentEntry = new TsRollingArgumentEntry(); - entryProto.getTsRecords().getTsRecordsMap().forEach((ts, objectProto) -> { - Object value = deserializeObjectProto(objectProto); - tsRollingArgumentEntry.getTsRecords().put(ts, value); - }); - argumentEntry = tsRollingArgumentEntry; - } else if (entryProto.hasSingleValue()) { - TransportProtos.SingleValueProto singleRecordProto = entryProto.getSingleValue(); - Object value = deserializeObjectProto(singleRecordProto.getValue()); - argumentEntry = new SingleValueArgumentEntry(singleRecordProto.getTs(), value); - } else { - throw new IllegalArgumentException("Unsupported ArgumentEntryProto type"); - } - argumentsMap.put(key, argumentEntry); - }); + List calculatedFieldIds = proto.getCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId(new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList(); + Map argumentsMap = proto.getArgumentsMap().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> fromArgumentEntryProto(entry.getValue()))); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, calculatedFieldIds); @@ -424,16 +446,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldCache.getEntitiesByProfile(tenantId, newProfileId).add(entityId); calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) - .forEach(cfId -> { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); - if (tpi.isMyPartition()) { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); - } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, Collections.emptyList(), null); - } - }); + .forEach(cfId -> clearState(tenantId, cfId, entityId)); initializeStateForEntityByProfile(tenantId, entityId, newProfileId, callback); } catch (Exception e) { @@ -455,16 +468,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream().map(CalculatedFieldLink::getCalculatedFieldId), calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream().map(CalculatedFieldLink::getCalculatedFieldId) ).toList(); - calculatedFieldIds.forEach(cfId -> { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); - if (tpi.isMyPartition()) { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); - } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, Collections.emptyList(), null); - } - }); + calculatedFieldIds.forEach(cfId -> clearState(tenantId, cfId, entityId)); } else { log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).add(entityId); @@ -475,94 +479,16 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List calculatedFieldIds, Map argumentValues) { - TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = TransportProtos.CalculatedFieldStateMsgProto.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) - .setEntityType(entityId.getEntityType().name()) - .setEntityIdMSB(entityId.getId().getMostSignificantBits()) - .setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - - if (argumentValues != null) { - argumentValues.forEach((key, argumentEntry) -> { - TransportProtos.ArgumentEntryProto.Builder argumentEntryProtoBuilder = TransportProtos.ArgumentEntryProto.newBuilder(); - - if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { - TransportProtos.TsRollingProto.Builder tsRollingProtoBuilder = TransportProtos.TsRollingProto.newBuilder(); - - tsRollingArgumentEntry.getTsRecords().forEach((ts, value) -> { - TransportProtos.ObjectProto.Builder objectProtoBuilder = TransportProtos.ObjectProto.newBuilder() - .setType(value.getClass().getName()) - .setValue(value.toString()); - tsRollingProtoBuilder.putTsRecords(ts, objectProtoBuilder.build()); - }); - - argumentEntryProtoBuilder.setTsRecords(tsRollingProtoBuilder.build()); - } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - TransportProtos.SingleValueProto.Builder singleRecordProtoBuilder = TransportProtos.SingleValueProto.newBuilder() - .setTs(singleValueArgumentEntry.getTs()) - .setValue(TransportProtos.ObjectProto.newBuilder() - .setType(singleValueArgumentEntry.getValue().getClass().getName()) - .setValue(singleValueArgumentEntry.getValue().toString()) - .build()); - argumentEntryProtoBuilder.setSingleValue(singleRecordProtoBuilder.build()); - } - - msgBuilder.putArguments(key, argumentEntryProtoBuilder.build()); - }); - } - - clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); - } - - private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); - boolean shouldReinit = true; - if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { - onCalculatedFieldDelete(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId(), callback); + private void clearState(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + if (tpi.isMyPartition()) { + log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId()); + states.remove(ctxId); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } else { - callback.onSuccess(); - shouldReinit = false; + sendClearCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); } - return shouldReinit; - } - - private void onCalculatedFieldDelete(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbCallback callback) { - try { - cleanupEntity(calculatedFieldId); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); - Set calculatedFieldIds = partitionedEntities.get(tpi); - if (calculatedFieldIds != null) { - calculatedFieldIds.remove(calculatedFieldId); - } - calculatedFieldCache.evict(calculatedFieldId); - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); - List statesToRemove = states.keySet().stream() - .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) - .map(JacksonUtil::writeValueAsString) - .toList(); - rocksDBService.deleteAll(statesToRemove); - } catch (Exception e) { - log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); - callback.onFailure(e); - } - } - - private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { - if (oldCalculatedField == null) { - return true; - } - boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); - boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); - CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); - CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); - boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); - boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); - boolean expressionChanged = !oldConfig.getExpression().equals(newConfig.getExpression()); - - return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; } private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { @@ -605,82 +531,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } - private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { - Map argumentValues = new HashMap<>(); - List> futures = new ArrayList<>(); - necessaryArguments.forEach((key, argument) -> { - futures.add(Futures.transform(fetchArgumentValue(tenantId, entityId, argument), - result -> { - argumentValues.put(key, result); - return result; - }, calculatedFieldCallbackExecutor)); - }); - return Futures.transform(Futures.allAsList(futures), results -> { - onComplete.accept(argumentValues); - return null; - }, calculatedFieldCallbackExecutor); - } - - private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { - EntityId argumentEntityId = argument.getEntityId(); - EntityId entityId = isProfileEntity(argumentEntityId) - ? targetEntityId - : argumentEntityId; - return fetchKvEntry(tenantId, entityId, argument); - } - - private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { - return switch (argument.getType()) { - case "TS_ROLLING" -> fetchTsRolling(tenantId, entityId, argument); - case "ATTRIBUTE" -> transformSingleValueArgument( - Futures.transform( - attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), - result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), - calculatedFieldCallbackExecutor) - ); - case "TS_LATEST" -> transformSingleValueArgument( - Futures.transform( - timeseriesService.findLatest(tenantId, entityId, argument.getKey()), - result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), - calculatedFieldCallbackExecutor)); - default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); - }; - } - - private ListenableFuture fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument) { - long currentTime = System.currentTimeMillis(); - long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); - long startTs = currentTime - timeWindow; - int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); - - ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); - ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); - - return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); - } - - private ListenableFuture transformSingleValueArgument(ListenableFuture> kvEntryFuture) { - return Futures.transform(kvEntryFuture, kvEntry -> { - if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { - return ArgumentEntry.createSingleValueArgument(kvEntry.get()); - } else { - return SingleValueArgumentEntry.EMPTY; - } - }, calculatedFieldCallbackExecutor); - } - - private KvEntry createDefaultKvEntry(Argument argument) { - String key = argument.getKey(); - String defaultValue = argument.getDefaultValue(); - if (NumberUtils.isParsable(defaultValue)) { - return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); - } - if ("true".equalsIgnoreCase(defaultValue) || "false".equalsIgnoreCase(defaultValue)) { - return new BooleanDataEntry(key, Boolean.parseBoolean(defaultValue)); - } - return new StringDataEntry(key, defaultValue); - } - private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List calculatedFieldIds) { TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); @@ -742,14 +592,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, MoreExecutors.directExecutor()); } - private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { - String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); - if (stateStr == null) { - return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); - } - return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); - } - private void pushMsgToRuleEngine(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult, List calculatedFieldIds) { try { String type = calculatedFieldResult.getType(); @@ -770,6 +612,166 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { + Map argumentValues = new HashMap<>(); + List> futures = new ArrayList<>(); + necessaryArguments.forEach((key, argument) -> { + futures.add(Futures.transform(fetchArgumentValue(tenantId, entityId, argument), + result -> { + argumentValues.put(key, result); + return result; + }, calculatedFieldCallbackExecutor)); + }); + return Futures.transform(Futures.allAsList(futures), results -> { + onComplete.accept(argumentValues); + return null; + }, calculatedFieldCallbackExecutor); + } + + private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { + EntityId argumentEntityId = argument.getEntityId(); + EntityId entityId = isProfileEntity(argumentEntityId) + ? targetEntityId + : argumentEntityId; + return fetchKvEntry(tenantId, entityId, argument); + } + + private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { + return switch (argument.getType()) { + case "TS_ROLLING" -> fetchTsRolling(tenantId, entityId, argument); + case "ATTRIBUTE" -> transformSingleValueArgument( + Futures.transform( + attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), + result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + calculatedFieldCallbackExecutor) + ); + case "TS_LATEST" -> transformSingleValueArgument( + Futures.transform( + timeseriesService.findLatest(tenantId, entityId, argument.getKey()), + result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + calculatedFieldCallbackExecutor)); + default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); + }; + } + + private ListenableFuture transformSingleValueArgument(ListenableFuture> kvEntryFuture) { + return Futures.transform(kvEntryFuture, kvEntry -> { + if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { + return ArgumentEntry.createSingleValueArgument(kvEntry.get()); + } else { + return SingleValueArgumentEntry.EMPTY; + } + }, calculatedFieldCallbackExecutor); + } + + private ListenableFuture fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument) { + long currentTime = System.currentTimeMillis(); + long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); + long startTs = currentTime - timeWindow; + int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); + + ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); + ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); + + return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); + } + + private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List calculatedFieldIds, Map argumentValues) { + TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = createBaseCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); + if (argumentValues != null) { + argumentValues.forEach((key, argumentEntry) -> msgBuilder.putArguments(key, toArgumentEntryProto(argumentEntry))); + } + if (calculatedFieldIds != null) { + calculatedFieldIds.forEach(cfId -> msgBuilder.addCalculatedFields( + TransportProtos.CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) + .build() + )); + } + + clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); + } + + private void sendClearCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId) { + TransportProtos.CalculatedFieldStateMsgProto msg = createBaseCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId) + .setClear(true) + .build(); + + clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msg).build(), null); + } + + private TransportProtos.CalculatedFieldStateMsgProto.Builder createBaseCalculatedFieldStateMsg( + TenantId tenantId, + CalculatedFieldId calculatedFieldId, + EntityId entityId + ) { + return TransportProtos.CalculatedFieldStateMsgProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + } + + private TransportProtos.ArgumentEntryProto toArgumentEntryProto(ArgumentEntry argumentEntry) { + TransportProtos.ArgumentEntryProto.Builder argumentProtoBuilder = TransportProtos.ArgumentEntryProto.newBuilder(); + + if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { + TransportProtos.TsRollingProto.Builder tsRollingProtoBuilder = TransportProtos.TsRollingProto.newBuilder(); + tsRollingArgumentEntry.getTsRecords().forEach((ts, value) -> + tsRollingProtoBuilder.putTsRecords(ts, toObjectProto(value)) + ); + argumentProtoBuilder.setTsRecords(tsRollingProtoBuilder.build()); + } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + argumentProtoBuilder.setSingleValue( + TransportProtos.SingleValueProto.newBuilder() + .setTs(singleValueArgumentEntry.getTs()) + .setValue(toObjectProto(singleValueArgumentEntry.getValue())) + .build() + ); + } + + return argumentProtoBuilder.build(); + } + + private ArgumentEntry fromArgumentEntryProto(TransportProtos.ArgumentEntryProto entryProto) { + if (entryProto.hasTsRecords()) { + TsRollingArgumentEntry tsRollingArgumentEntry = new TsRollingArgumentEntry(); + entryProto.getTsRecords().getTsRecordsMap().forEach((ts, objectProto) -> + tsRollingArgumentEntry.getTsRecords().put(ts, fromObjectProto(objectProto)) + ); + return tsRollingArgumentEntry; + } else if (entryProto.hasSingleValue()) { + TransportProtos.SingleValueProto singleValueProto = entryProto.getSingleValue(); + return new SingleValueArgumentEntry(singleValueProto.getTs(), fromObjectProto(singleValueProto.getValue())); + } else { + throw new IllegalArgumentException("Unsupported ArgumentEntryProto type"); + } + } + + private KvEntry createDefaultKvEntry(Argument argument) { + String key = argument.getKey(); + String defaultValue = argument.getDefaultValue(); + if (NumberUtils.isParsable(defaultValue)) { + return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); + } + if ("true".equalsIgnoreCase(defaultValue) || "false".equalsIgnoreCase(defaultValue)) { + return new BooleanDataEntry(key, Boolean.parseBoolean(defaultValue)); + } + return new StringDataEntry(key, defaultValue); + } + + private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { + String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); + if (stateStr == null) { + return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); + } + return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); + } + private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { ObjectNode payload = JacksonUtil.newObjectNode(); Map resultMap = calculatedFieldResult.getResultMap(); diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 264b23f118..ec17914fd8 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -1183,6 +1183,46 @@ public class ProtoUtils { return builder.build(); } + public static TransportProtos.ObjectProto toObjectProto(Object value) { + if (value == null) { + throw new IllegalArgumentException("Cannot convert null to ObjectProto"); + } + + TransportProtos.ObjectProto.Builder builder = TransportProtos.ObjectProto.newBuilder(); + + if (value instanceof String) { + builder.setStringValue((String) value); + } else if (value instanceof Integer) { + builder.setIntValue((Integer) value); + } else if (value instanceof Long) { + builder.setLongValue((Long) value); + } else if (value instanceof Double) { + builder.setDoubleValue((Double) value); + } else if (value instanceof Boolean) { + builder.setBoolValue((Boolean) value); + } else { + throw new IllegalArgumentException("Unsupported value type: " + value.getClass().getName()); + } + + return builder.build(); + } + + public static Object fromObjectProto(TransportProtos.ObjectProto proto) { + try { + return switch (proto.getValueCase()) { + case STRINGVALUE -> proto.getStringValue(); + case INTVALUE -> proto.getIntValue(); + case LONGVALUE -> proto.getLongValue(); + case DOUBLEVALUE -> proto.getDoubleValue(); + case BOOLVALUE -> proto.getBoolValue(); + case VALUE_NOT_SET -> throw new IllegalArgumentException("Value not set in ObjectProto"); + }; + } catch (Exception e) { + log.error("Failed to deserialize ObjectProto: [{}]", proto, e); + return null; + } + } + private static boolean isNotNull(Object obj) { return obj != null; } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index f6f7afd5d8..581f9eb9f2 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -817,8 +817,9 @@ message CalculatedFieldStateMsgProto { string entityType = 5; int64 entityIdMSB = 6; int64 entityIdLSB = 7; - repeated CalculatedFieldIdProto calculatedFields = 8; - map arguments = 9; + bool clear = 8; + repeated CalculatedFieldIdProto calculatedFields = 9; + map arguments = 10; } message CalculatedFieldIdProto { @@ -843,8 +844,13 @@ message SingleValueProto { } message ObjectProto { - string type = 1; - string value = 2; + oneof value { + string stringValue = 1; + int32 intValue = 2; + int64 longValue = 3; + double doubleValue = 4; + bool boolValue = 5; + } } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. From befd4cc9c69749dc510c5ff6ae2ff67713264d06 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 26 Dec 2024 10:56:20 +0200 Subject: [PATCH 10/14] added check for update values for ts rolling when rocksdb fails --- .../service/cf/DefaultCalculatedFieldCache.java | 8 ++++---- .../cf/DefaultCalculatedFieldExecutionService.java | 13 +++++++------ .../cf/ctx/state/BaseCalculatedFieldState.java | 1 + 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 1fc2a90e07..dd2ab3857c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -100,11 +100,11 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { List cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null) { + if (cfLinks == null || cfLinks.isEmpty()) { calculatedFieldFetchLock.lock(); try { cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null) { + if (cfLinks == null || cfLinks.isEmpty()) { cfLinks = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); if (cfLinks != null) { calculatedFieldLinks.put(calculatedFieldId, cfLinks); @@ -122,11 +122,11 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { List cfLinks = entityIdCalculatedFieldLinks.get(entityId); - if (cfLinks == null) { + if (cfLinks == null || cfLinks.isEmpty()) { calculatedFieldFetchLock.lock(); try { cfLinks = entityIdCalculatedFieldLinks.get(entityId); - if (cfLinks == null) { + if (cfLinks == null || cfLinks.isEmpty()) { cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); if (cfLinks != null) { entityIdCalculatedFieldLinks.put(entityId, cfLinks); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 6e7536825a..808c1ea8d8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -556,20 +556,21 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldState state = calculatedFieldEntityCtx.getState(); boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); - if (!allKeysPresent) { + boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() + .anyMatch(argument -> "TS_ROLLING".equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); + + if (!allKeysPresent || requiresTsRollingUpdate) { Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentValues.containsKey(entry.getKey())) + .filter(entry -> !argumentValues.containsKey(entry.getKey()) || ("TS_ROLLING".equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) .addListener(() -> performUpdateState.accept(state), calculatedFieldCallbackExecutor); - return; + } else { + performUpdateState.accept(state); } - performUpdateState.accept(state); - states.put(entityCtxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); } else { sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentValues); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index b73ac51798..462fa19b17 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -24,6 +24,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { protected Map arguments; public BaseCalculatedFieldState() { + arguments = new HashMap<>(); } @Override From e787b805dae819f083876168e0a960bd5c801ae1 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 26 Dec 2024 17:05:27 +0200 Subject: [PATCH 11/14] added locking and logic to avoid iteration order issue in map --- ...efaultCalculatedFieldExecutionService.java | 68 +++++++++++-------- .../service/cf/ctx/state/ArgumentEntry.java | 5 +- .../ctx/state/BaseCalculatedFieldState.java | 49 ++++++------- .../cf/ctx/state/CalculatedFieldCtx.java | 8 ++- .../ctx/state/ScriptCalculatedFieldState.java | 4 +- .../ctx/state/SingleValueArgumentEntry.java | 14 ++-- .../cf/ctx/state/TsRollingArgumentEntry.java | 8 ++- common/proto/src/main/proto/queue.proto | 1 + 8 files changed, 87 insertions(+), 70 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 808c1ea8d8..af4b3e95e5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -92,6 +92,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -119,6 +120,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; + private final ConcurrentMap entityLocks = new ConcurrentHashMap<>(); + private final ConcurrentMap states = new ConcurrentHashMap<>(); private static final int MAX_LAST_RECORDS_VALUE = 1024; @@ -536,40 +539,47 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); if (tpi.isMyPartition()) { - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); - - Consumer performUpdateState = (state) -> { - if (state.updateState(argumentValues)) { - calculatedFieldEntityCtx.setState(state); - states.put(entityCtxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - Map arguments = state.getArguments(); - boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && - !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); - if (allArgsPresent) { - performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); + ReentrantLock lock = entityLocks.computeIfAbsent(entityId, id -> new ReentrantLock()); + lock.lock(); + + try { + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); + + Consumer performUpdateState = (state) -> { + if (state.updateState(argumentValues)) { + calculatedFieldEntityCtx.setState(state); + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + Map arguments = state.getArguments(); + boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && + !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); + if (allArgsPresent) { + performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); + } } - } - }; + }; - CalculatedFieldState state = calculatedFieldEntityCtx.getState(); + CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); - boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() - .anyMatch(argument -> "TS_ROLLING".equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); + boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); + boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() + .anyMatch(argument -> "TS_ROLLING".equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); - if (!allKeysPresent || requiresTsRollingUpdate) { + if (!allKeysPresent || requiresTsRollingUpdate) { - Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentValues.containsKey(entry.getKey()) || ("TS_ROLLING".equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() + .filter(entry -> !argumentValues.containsKey(entry.getKey()) || ("TS_ROLLING".equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) - .addListener(() -> performUpdateState.accept(state), - calculatedFieldCallbackExecutor); - } else { - performUpdateState.accept(state); + fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) + .addListener(() -> performUpdateState.accept(state), + calculatedFieldCallbackExecutor); + } else { + performUpdateState.accept(state); + } + } finally { + lock.unlock(); } } else { sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentValues); @@ -747,7 +757,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return tsRollingArgumentEntry; } else if (entryProto.hasSingleValue()) { TransportProtos.SingleValueProto singleValueProto = entryProto.getSingleValue(); - return new SingleValueArgumentEntry(singleValueProto.getTs(), fromObjectProto(singleValueProto.getValue())); + return new SingleValueArgumentEntry(singleValueProto.getTs(), fromObjectProto(singleValueProto.getValue()), singleValueProto.getVersion()); } else { throw new IllegalArgumentException("Unsupported ArgumentEntryProto type"); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index ba7e094f77..dd56405352 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -22,8 +22,6 @@ import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.List; -import java.util.TreeMap; -import java.util.stream.Collectors; @JsonTypeInfo( use = JsonTypeInfo.Id.NAME, @@ -48,8 +46,7 @@ public interface ArgumentEntry { } static ArgumentEntry createTsRollingArgument(List kvEntries) { - return new TsRollingArgumentEntry(kvEntries.stream(). - collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue, (oldValue, newValue) -> newValue, TreeMap::new))); + return new TsRollingArgumentEntry(kvEntries); } @JsonIgnore diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 462fa19b17..be7319b930 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf.ctx.state; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; public abstract class BaseCalculatedFieldState implements CalculatedFieldState { @@ -37,34 +36,30 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { if (arguments == null) { arguments = new HashMap<>(); } - AtomicBoolean stateUpdated = new AtomicBoolean(false); - argumentValues.forEach((key, argumentEntry) -> { - ArgumentEntry existingArgumentEntry = arguments.get(key); - if (existingArgumentEntry != null) { - if (existingArgumentEntry instanceof SingleValueArgumentEntry) { - if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - arguments.put(key, argumentEntry.copy()); - stateUpdated.set(true); - } - } else if (existingArgumentEntry instanceof TsRollingArgumentEntry existingTsRollingArgumentEntry) { - if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { - if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - existingTsRollingArgumentEntry.addAllTsRecords(tsRollingArgumentEntry.getTsRecords()); - stateUpdated.set(true); - } - } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - existingTsRollingArgumentEntry.addTsRecord(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); - stateUpdated.set(true); - } - } + + boolean stateUpdated = false; + + for (Map.Entry entry : argumentValues.entrySet()) { + String key = entry.getKey(); + ArgumentEntry newEntry = entry.getValue(); + ArgumentEntry existingEntry = arguments.get(key); + + if (existingEntry == null || existingEntry.hasUpdatedValue(newEntry)) { + if (existingEntry instanceof TsRollingArgumentEntry existingTsRollingEntry && newEntry instanceof TsRollingArgumentEntry newTsRollingEntry) { + existingTsRollingEntry.addAllTsRecords(newTsRollingEntry.getTsRecords()); + } else if (existingEntry instanceof TsRollingArgumentEntry existingTsRollingEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry) { + existingTsRollingEntry.addTsRecord(singleValueEntry.getTs(), singleValueEntry.getValue()); + } else if (existingEntry instanceof SingleValueArgumentEntry existingSingleValueEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry + && singleValueEntry.getVersion() > existingSingleValueEntry.getVersion()) { + arguments.put(key, newEntry.copy()); + } else { + arguments.put(key, newEntry.copy()); } - } else { - arguments.put(key, argumentEntry.copy()); - stateUpdated.set(true); + stateUpdated = true; } - }); - return stateUpdated.get(); + } + + return stateUpdated; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 2cd5c68144..d54a3220ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -26,6 +26,8 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import java.util.ArrayList; +import java.util.List; import java.util.Map; @Data @@ -36,6 +38,7 @@ public class CalculatedFieldCtx { private EntityId entityId; private CalculatedFieldType cfType; private final Map arguments; + private final List argKeys; private Output output; private String expression; private TbelInvokeService tbelInvokeService; @@ -48,10 +51,11 @@ public class CalculatedFieldCtx { this.cfType = calculatedField.getType(); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); this.arguments = configuration.getArguments(); + this.argKeys = new ArrayList<>(arguments.keySet()); this.output = configuration.getOutput(); this.expression = configuration.getExpression(); this.tbelInvokeService = tbelInvokeService; - if (!CalculatedFieldType.SIMPLE.equals(calculatedField.getType())) { + if (CalculatedFieldType.SCRIPT.equals(calculatedField.getType())) { this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); } } @@ -65,7 +69,7 @@ public class CalculatedFieldCtx { tenantId, tbelInvokeService, expression, - arguments.keySet().toArray(new String[0]) + argKeys.toArray(String[]::new) ); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 87429050de..de7c514786 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -49,7 +49,9 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); } }); - Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); + Object[] args = ctx.getArgKeys().stream() + .map(key -> arguments.get(key).getValue()) + .toArray(); ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); Output output = ctx.getOutput(); return Futures.transform(resultFuture, diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 3f4fd5bdce..20b531f562 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -32,11 +32,15 @@ public class SingleValueArgumentEntry implements ArgumentEntry { private long ts; private Object value; + private long version; + public SingleValueArgumentEntry(KvEntry entry) { - if (entry instanceof TsKvEntry) { - this.ts = ((TsKvEntry) entry).getTs(); - } else if (entry instanceof AttributeKvEntry) { - this.ts = ((AttributeKvEntry) entry).getLastUpdateTs(); + if (entry instanceof TsKvEntry tsKvEntry) { + this.ts = tsKvEntry.getTs(); + this.version = tsKvEntry.getVersion(); + } else if (entry instanceof AttributeKvEntry attributeKvEntry) { + this.ts = attributeKvEntry.getLastUpdateTs(); + this.version = attributeKvEntry.getVersion(); } this.value = entry.getValue(); } @@ -66,7 +70,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { @Override public ArgumentEntry copy() { - return new SingleValueArgumentEntry(this.ts, this.value); + return new SingleValueArgumentEntry(this.ts, this.value, this.version); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 4ffa391550..64de2f8c8a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -16,16 +16,20 @@ package org.thingsboard.server.service.cf.ctx.state; import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.math.NumberUtils; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import java.util.List; import java.util.Map; import java.util.TreeMap; @Data @NoArgsConstructor +@AllArgsConstructor @Slf4j public class TsRollingArgumentEntry implements ArgumentEntry { @@ -35,8 +39,8 @@ public class TsRollingArgumentEntry implements ArgumentEntry { private TreeMap tsRecords = new TreeMap<>(); - public TsRollingArgumentEntry(TreeMap tsRecords) { - addAllTsRecords(tsRecords); + public TsRollingArgumentEntry(List kvEntries) { + kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry.getValue())); } /** diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 581f9eb9f2..1f66621a5b 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -841,6 +841,7 @@ message TsRollingProto { message SingleValueProto { int64 ts = 1; ObjectProto value = 2; + int64 version = 3; } message ObjectProto { From 5f088a751e2fc18435fecfeee603c592ef412714 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 27 Dec 2024 18:20:20 +0200 Subject: [PATCH 12/14] refactored code --- .../cf/CalculatedFieldExecutionService.java | 9 +- .../service/cf/CalculatedFieldResult.java | 5 +- ...efaultCalculatedFieldExecutionService.java | 122 ++++++++++-------- .../service/cf/ctx/state/ArgumentEntry.java | 2 +- ...gumentType.java => ArgumentEntryType.java} | 2 +- .../ctx/state/BaseCalculatedFieldState.java | 8 +- .../ctx/state/SingleValueArgumentEntry.java | 6 +- .../cf/ctx/state/TsRollingArgumentEntry.java | 4 +- ...CalculatedFieldAttributeUpdateRequest.java | 38 ++++++ ...CalculatedFieldTelemetryUpdateRequest.java | 38 ++++++ ...alculatedFieldTimeSeriesUpdateRequest.java | 42 ++++++ .../DefaultTelemetrySubscriptionService.java | 6 +- .../CalculatedFieldControllerTest.java | 6 +- .../cf/CalculatedFieldLinkConfiguration.java | 4 +- .../data/cf/configuration/Argument.java | 2 +- .../data/cf/configuration/ArgumentType.java | 22 ++++ .../BaseCalculatedFieldConfiguration.java | 45 ++++--- .../common/data/cf/configuration/Output.java | 2 +- .../data/cf/configuration/OutputType.java | 22 ++++ .../server/dao/service/AssetServiceTest.java | 6 +- .../service/CalculatedFieldServiceTest.java | 6 +- .../dao/service/CustomerServiceTest.java | 6 +- .../server/dao/service/DeviceServiceTest.java | 6 +- 23 files changed, 302 insertions(+), 107 deletions(-) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/{ArgumentType.java => ArgumentEntryType.java} (95%) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index ee9b6d115c..e4b0a7ca1e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,20 +15,15 @@ */ package org.thingsboard.server.service.cf; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos; - -import java.util.List; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List calculatedFieldIds, List telemetry); + void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest); void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index e8ea318bf6..52e0a0151b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -17,17 +17,18 @@ package org.thingsboard.server.service.cf; import lombok.Data; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import java.util.Map; @Data public final class CalculatedFieldResult { - private final String type; + private final OutputType type; private final AttributeScope scope; private final Map resultMap; - public CalculatedFieldResult(String type, AttributeScope scope, Map resultMap) { + public CalculatedFieldResult(OutputType type, AttributeScope scope, Map resultMap) { this.type = type; this.scope = scope; this.resultMap = resultMap == null ? Map.of() : Map.copyOf(resultMap); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index af4b3e95e5..27db9ae290 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -35,12 +35,15 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; @@ -48,7 +51,6 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; @@ -79,11 +81,13 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -92,7 +96,6 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -120,12 +123,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; - private final ConcurrentMap entityLocks = new ConcurrentHashMap<>(); - private final ConcurrentMap states = new ConcurrentHashMap<>(); private static final int MAX_LAST_RECORDS_VALUE = 1024; + private static final Set supportedReferencedEntities = EnumSet.of( + EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT + ); + @Value("${calculatedField.initFetchPackSize:50000}") @Getter private int initFetchPackSize; @@ -336,28 +341,29 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List calculatedFieldIds, List telemetry) { + public void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest) { try { - EntityType entityType = entityId.getEntityType(); - if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType) || EntityType.CUSTOMER.equals(entityType) || EntityType.TENANT.equals(entityType)) { - EntityId profileId = null; - if (EntityType.ASSET.equals(entityType)) { - profileId = assetProfileCache.get(tenantId, (AssetId) entityId).getId(); - } else if (EntityType.DEVICE.equals(entityType)) { - profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); - } - List cfLinks = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId)); - Optional.ofNullable(profileId).ifPresent(id -> { - cfLinks.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, id)); - }); + TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); + EntityId entityId = calculatedFieldTelemetryUpdateRequest.getEntityId(); + AttributeScope scope = calculatedFieldTelemetryUpdateRequest.getScope(); + List telemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries(); + List calculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getCalculatedFieldIds(); + + if (supportedReferencedEntities.contains(entityId.getEntityType())) { + EntityId profileId = getProfileId(tenantId, entityId); + + List cfLinks = Stream.concat( + calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream(), + profileId != null ? calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream() : Stream.empty() + ).toList(); + cfLinks.forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - Map attributes = link.getConfiguration().getAttributes(); - Map timeSeries = link.getConfiguration().getTimeSeries(); + Map telemetryKeys = getTelemetryKeysFromLink(link, scope); Map updatedTelemetry = telemetry.stream() - .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) + .filter(entry -> telemetryKeys.containsValue(entry.getKey())) .collect(Collectors.toMap( - entry -> getMappedKey(entry, attributes, timeSeries), + entry -> getMappedKey(entry, telemetryKeys), entry -> entry, (v1, v2) -> v1 )); @@ -368,25 +374,24 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }); } } catch (Exception e) { - log.trace("Failed to update telemetry entityId: [{}]", entityId, e); + log.trace("Failed to update telemetry.", e); } } - private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { - if (entry instanceof AttributeKvEntry) { - return attributes.entrySet().stream() - .filter(attr -> attr.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } else if (entry instanceof TsKvEntry) { - return timeSeries.entrySet().stream() - .filter(ts -> ts.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } - return entry.getKey(); + private Map getTelemetryKeysFromLink(CalculatedFieldLink link, AttributeScope scope) { + return scope == null ? link.getConfiguration().getTimeSeries() : switch (scope) { + case CLIENT_SCOPE -> link.getConfiguration().getClientAttributes(); + case SERVER_SCOPE -> link.getConfiguration().getServerAttributes(); + case SHARED_SCOPE -> link.getConfiguration().getSharedAttributes(); + }; + } + + private String getMappedKey(KvEntry entry, Map telemetry) { + return telemetry.entrySet().stream() + .filter(kvEntry -> kvEntry.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); } private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List calculatedFieldIds, Map updatedTelemetry) { @@ -539,17 +544,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); if (tpi.isMyPartition()) { - ReentrantLock lock = entityLocks.computeIfAbsent(entityId, id -> new ReentrantLock()); - lock.lock(); + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - try { - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); + states.compute(entityCtxId, (ctxId, ctx) -> { + CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); Consumer performUpdateState = (state) -> { if (state.updateState(argumentValues)) { calculatedFieldEntityCtx.setState(state); - states.put(entityCtxId, calculatedFieldEntityCtx); rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); Map arguments = state.getArguments(); boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && @@ -564,12 +566,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() - .anyMatch(argument -> "TS_ROLLING".equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); + .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); if (!allKeysPresent || requiresTsRollingUpdate) { Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentValues.containsKey(entry.getKey()) || ("TS_ROLLING".equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) + .filter(entry -> !argumentValues.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) @@ -578,9 +580,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } else { performUpdateState.accept(state); } - } finally { - lock.unlock(); - } + return calculatedFieldEntityCtx; + }); } else { sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentValues); } @@ -605,9 +606,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void pushMsgToRuleEngine(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult, List calculatedFieldIds) { try { - String type = calculatedFieldResult.getType(); - TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; - TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; + OutputType type = calculatedFieldResult.getType(); + TbMsgType msgType = OutputType.ATTRIBUTES.equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; + TbMsgMetaData md = OutputType.ATTRIBUTES.equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); if (calculatedFieldIds == null) { calculatedFieldIds = new ArrayList<>(); @@ -649,19 +650,18 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { return switch (argument.getType()) { - case "TS_ROLLING" -> fetchTsRolling(tenantId, entityId, argument); - case "ATTRIBUTE" -> transformSingleValueArgument( + case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument); + case ATTRIBUTE -> transformSingleValueArgument( Futures.transform( attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), - result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(createDefaultKvEntry(argument), System.currentTimeMillis(), 0L))), calculatedFieldCallbackExecutor) ); - case "TS_LATEST" -> transformSingleValueArgument( + case TS_LATEST -> transformSingleValueArgument( Futures.transform( timeseriesService.findLatest(tenantId, entityId, argument.getKey()), - result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument), 0L))), calculatedFieldCallbackExecutor)); - default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; } @@ -801,4 +801,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return EntityType.DEVICE_PROFILE.equals(entityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(entityId.getEntityType()); } + private EntityId getProfileId(TenantId tenantId, EntityId entityId) { + return switch (entityId.getEntityType()) { + case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); + default -> null; + }; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index dd56405352..b261840bfd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -35,7 +35,7 @@ import java.util.List; public interface ArgumentEntry { @JsonIgnore - ArgumentType getType(); + ArgumentEntryType getType(); Object getValue(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java index 360529a7e9..1a0dfb5ac7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java @@ -15,6 +15,6 @@ */ package org.thingsboard.server.service.cf.ctx.state; -public enum ArgumentType { +public enum ArgumentEntryType { SINGLE_VALUE, TS_ROLLING } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index be7319b930..bc9a421e47 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -49,8 +49,12 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { existingTsRollingEntry.addAllTsRecords(newTsRollingEntry.getTsRecords()); } else if (existingEntry instanceof TsRollingArgumentEntry existingTsRollingEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry) { existingTsRollingEntry.addTsRecord(singleValueEntry.getTs(), singleValueEntry.getValue()); - } else if (existingEntry instanceof SingleValueArgumentEntry existingSingleValueEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry - && singleValueEntry.getVersion() > existingSingleValueEntry.getVersion()) { + } else if (existingEntry instanceof SingleValueArgumentEntry existingSingleValueEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry) { +// Long existingVersion = existingSingleValueEntry.getVersion(); +// Long newVersion = singleValueEntry.getVersion(); +// if (newVersion != null && (existingVersion == null || newVersion > existingVersion)) { +// arguments.put(key, newEntry.copy()); +// } arguments.put(key, newEntry.copy()); } else { arguments.put(key, newEntry.copy()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 20b531f562..8d5080e90f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -32,7 +32,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { private long ts; private Object value; - private long version; + private Long version; public SingleValueArgumentEntry(KvEntry entry) { if (entry instanceof TsKvEntry tsKvEntry) { @@ -54,8 +54,8 @@ public class SingleValueArgumentEntry implements ArgumentEntry { } @Override - public ArgumentType getType() { - return ArgumentType.SINGLE_VALUE; + public ArgumentEntryType getType() { + return ArgumentEntryType.SINGLE_VALUE; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 64de2f8c8a..1118e3af13 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -51,8 +51,8 @@ public class TsRollingArgumentEntry implements ArgumentEntry { } @Override - public ArgumentType getType() { - return ArgumentType.TS_ROLLING; + public ArgumentEntryType getType() { + return ArgumentEntryType.TS_ROLLING; } @JsonIgnore diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java new file mode 100644 index 0000000000..8479ff37d7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.telemetry; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; + +import java.util.List; + +@Data +@AllArgsConstructor +public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { + + private TenantId tenantId; + private EntityId entityId; + private AttributeScope scope; + private List kvEntries; + private List calculatedFieldIds; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java new file mode 100644 index 0000000000..3c28833f31 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.telemetry; + +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; + +import java.util.List; + +public interface CalculatedFieldTelemetryUpdateRequest { + + TenantId getTenantId(); + + EntityId getEntityId(); + + AttributeScope getScope(); + + List getKvEntries(); + + List getCalculatedFieldIds(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java new file mode 100644 index 0000000000..987d899465 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.telemetry; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +@Data +@AllArgsConstructor +public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { + + private TenantId tenantId; + private EntityId entityId; + private List kvEntries; + private List calculatedFieldIds; + + @Override + public AttributeScope getScope() { + return null; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index e35d3cead6..bf3076be5c 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -50,6 +50,8 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldAttributeUpdateRequest; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTimeSeriesUpdateRequest; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -152,7 +154,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } - calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, request.getCalculatedFieldIds(), request.getEntries()); + calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getCalculatedFieldIds())); return saveFuture; } @@ -168,7 +170,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); - calculatedFieldExecutionService.onTelemetryUpdate(request.getTenantId(), request.getEntityId(), request.getCalculatedFieldIds(), request.getEntries()); + calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries(), request.getCalculatedFieldIds())); } @Override diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index ba1dfb1fec..5d1467974d 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -24,8 +24,10 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -140,7 +142,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { Argument argument = new Argument(); argument.setEntityId(referencedEntityId); - argument.setType("TS_LATEST"); + argument.setType(ArgumentType.TS_LATEST); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -149,7 +151,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { Output output = new Output(); output.setName("output"); - output.setType("TS_LATEST"); + output.setType(OutputType.TIME_SERIES); config.setOutput(output); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java index c5f81cd572..78513c8b74 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java @@ -23,7 +23,9 @@ import java.util.Map; @Data public class CalculatedFieldLinkConfiguration { - private Map attributes = new HashMap<>(); + private Map clientAttributes = new HashMap<>(); + private Map serverAttributes = new HashMap<>(); + private Map sharedAttributes = new HashMap<>(); private Map timeSeries = new HashMap<>(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index f34f5e9cb7..4dac866219 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -24,7 +24,7 @@ public class Argument { private EntityId entityId; private String key; - private String type; + private ArgumentType type; private AttributeScope scope; private String defaultValue; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java new file mode 100644 index 0000000000..17e2315b52 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +public enum ArgumentType { + + TS_LATEST, ATTRIBUTE, TS_ROLLING + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index ac36991a61..8c86b6c552 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -65,19 +65,24 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel public CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId) { CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); - for (Map.Entry entry : arguments.entrySet()) { - Argument argument = entry.getValue(); - if (argument.getEntityId().equals(entityId)) { - switch (argument.getType()) { - case "ATTRIBUTE": - linkConfiguration.getAttributes().put(entry.getKey(), argument.getKey()); - break; - case "TS_LATEST", "TS_ROLLING": - linkConfiguration.getTimeSeries().put(entry.getKey(), argument.getKey()); - break; - } - } - } + arguments.entrySet().stream() + .filter(entry -> entry.getValue().getEntityId().equals(entityId)) + .forEach(entry -> { + Argument argument = entry.getValue(); + String argumentKey = entry.getKey(); + + switch (argument.getType()) { + case ATTRIBUTE -> { + switch (argument.getScope()) { + case CLIENT_SCOPE -> linkConfiguration.getClientAttributes().put(entry.getKey(), argument.getKey()); + case SERVER_SCOPE -> linkConfiguration.getServerAttributes().put(entry.getKey(), argument.getKey()); + case SHARED_SCOPE -> linkConfiguration.getSharedAttributes().put(entry.getKey(), argument.getKey()); + } + } + case TS_LATEST, TS_ROLLING -> + linkConfiguration.getTimeSeries().put(argumentKey, argument.getKey()); + } + }); return linkConfiguration; } @@ -98,7 +103,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argumentNode.put("entityId", entityId.toString()); } argumentNode.put("key", argument.getKey()); - argumentNode.put("type", argument.getType()); + argumentNode.put("type", String.valueOf(argument.getType())); argumentNode.put("scope", String.valueOf(argument.getScope())); argumentNode.put("defaultValue", argument.getDefaultValue()); argumentNode.put("limit", String.valueOf(argument.getLimit())); @@ -112,7 +117,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel if (output != null) { ObjectNode outputNode = configNode.putObject("output"); outputNode.put("name", output.getName()); - outputNode.put("type", output.getType()); + outputNode.put("type", String.valueOf(output.getType())); if (output.getScope() != null) { outputNode.put("scope", String.valueOf(output.getScope())); } @@ -141,7 +146,10 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); } argument.setKey(argumentNode.get("key").asText()); - argument.setType(argumentNode.get("type").asText()); + JsonNode type = argumentNode.get("type"); + if (type != null && !type.isNull() && !type.asText().equals("null")) { + argument.setType(ArgumentType.valueOf(type.asText())); + } JsonNode scope = argumentNode.get("scope"); if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { argument.setScope(AttributeScope.valueOf(scope.asText())); @@ -169,7 +177,10 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel if (outputNode != null) { Output output = new Output(); output.setName(outputNode.get("name").asText()); - output.setType(outputNode.get("type").asText()); + JsonNode type = outputNode.get("type"); + if (type != null && !type.isNull() && !type.asText().equals("null")) { + output.setType(OutputType.valueOf(type.asText())); + } JsonNode scope = outputNode.get("scope"); if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { output.setScope(AttributeScope.valueOf(scope.asText())); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java index 46257d1ccc..12cf97338a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java @@ -22,7 +22,7 @@ import org.thingsboard.server.common.data.AttributeScope; public class Output { private String name; - private String type; + private OutputType type; private AttributeScope scope; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java new file mode 100644 index 0000000000..c248bc8042 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +public enum OutputType { + + TIME_SERIES, ATTRIBUTES + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index b0870f3dc5..9a0b9222f0 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -33,7 +33,9 @@ import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -884,7 +886,7 @@ public class AssetServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedAsset.getId()); - argument.setType("TS_LATEST"); + argument.setType(ArgumentType.TS_LATEST); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -893,7 +895,7 @@ public class AssetServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TS_LATEST"); + output.setType(OutputType.TIME_SERIES); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 9a1719e715..5a8f7a2383 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -26,8 +26,10 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -153,7 +155,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(referencedEntityId); - argument.setType("TS_LATEST"); + argument.setType(ArgumentType.TS_LATEST); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -162,7 +164,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TS_LATEST"); + output.setType(OutputType.TIME_SERIES); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 6671e0e821..d0ee833261 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -34,7 +34,9 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -379,7 +381,7 @@ public class CustomerServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedCustomer.getId()); - argument.setType("TS_LATEST"); + argument.setType(ArgumentType.TS_LATEST); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -388,7 +390,7 @@ public class CustomerServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TS_LATEST"); + output.setType(OutputType.TIME_SERIES); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index f2f8686bc3..5b060ae145 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -42,7 +42,9 @@ import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -1222,7 +1224,7 @@ public class DeviceServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(device.getId()); - argument.setType("TS_LATEST"); + argument.setType(ArgumentType.TS_LATEST); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -1231,7 +1233,7 @@ public class DeviceServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TS_LATEST"); + output.setType(OutputType.TIME_SERIES); config.setOutput(output); From 8f87caba148aac97973b0d0f212d998957eb5a00 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 30 Dec 2024 16:50:01 +0200 Subject: [PATCH 13/14] implemented logic to handle profile updates even if no cf exists by them --- ...efaultCalculatedFieldExecutionService.java | 11 +++--- .../queue/DefaultTbClusterService.java | 36 ++++--------------- 2 files changed, 12 insertions(+), 35 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 27db9ae290..96cbdfc73b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -542,6 +542,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List calculatedFieldIds) { TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); + Map argumentsMap = new HashMap<>(argumentValues); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); if (tpi.isMyPartition()) { CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); @@ -550,7 +551,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); Consumer performUpdateState = (state) -> { - if (state.updateState(argumentValues)) { + if (state.updateState(argumentsMap)) { calculatedFieldEntityCtx.setState(state); rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); Map arguments = state.getArguments(); @@ -564,17 +565,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); + boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); if (!allKeysPresent || requiresTsRollingUpdate) { Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentValues.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) + .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) + fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) .addListener(() -> performUpdateState.accept(state), calculatedFieldCallbackExecutor); } else { @@ -583,7 +584,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return calculatedFieldEntityCtx; }); } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentValues); + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentsMap); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index d57a5266bf..6316d1fdb2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -68,7 +68,6 @@ import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.util.ProtoUtils; -import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; @@ -150,7 +149,6 @@ public class DefaultTbClusterService implements TbClusterService { private final GatewayNotificationsService gatewayNotificationsService; private final EdgeService edgeService; private final TbTransactionalCache edgeIdServiceIdCache; - private final CalculatedFieldService calculatedFieldService; @Override public void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback) { @@ -393,7 +391,7 @@ public class DefaultTbClusterService implements TbClusterService { public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) { DeviceId deviceId = device.getId(); gatewayNotificationsService.onDeviceDeleted(device); - handleEntityDelete(tenantId, deviceId, device.getDeviceProfileId()); + sendProfileEntityEvent(tenantId, deviceId, device.getDeviceProfileId(), false, true); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); @@ -402,17 +400,10 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { AssetId assetId = asset.getId(); - handleEntityDelete(tenantId, assetId, asset.getAssetProfileId()); + sendProfileEntityEvent(tenantId, assetId, asset.getAssetProfileId(), false, true); broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); } - private void handleEntityDelete(TenantId tenantId, EntityId entityId, EntityId profileId) { - boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); - if (cfExistsByProfile) { - sendProfileEntityEvent(tenantId, entityId, profileId, false, true); - } - } - @Override public void onDeviceAssignedToTenant(TenantId oldTenantId, Device device) { onDeviceDeleted(oldTenantId, device, null); @@ -633,13 +624,13 @@ public class DefaultTbClusterService implements TbClusterService { } boolean deviceTypeChanged = !device.getType().equals(old.getType()); if (deviceTypeChanged) { - handleProfileUpdate(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); + sendEntityProfileUpdatedEvent(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); } if (deviceNameChanged || deviceTypeChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); } } else { - handleEntityCreate(device.getTenantId(), device.getId(), device.getDeviceProfileId()); + sendProfileEntityEvent(device.getTenantId(), device.getId(), device.getDeviceProfileId(), true, false); } broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), created, !created, false); @@ -653,29 +644,14 @@ public class DefaultTbClusterService implements TbClusterService { if (old != null) { boolean assetTypeChanged = !asset.getType().equals(old.getType()); if (assetTypeChanged) { - handleProfileUpdate(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); + sendEntityProfileUpdatedEvent(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); } } else { - handleEntityCreate(asset.getTenantId(), asset.getId(), asset.getAssetProfileId()); + sendProfileEntityEvent(asset.getTenantId(), asset.getId(), asset.getAssetProfileId(), true, false); } broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } - private void handleProfileUpdate(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { - boolean cfExistsByOldProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, oldProfileId); - boolean cfExistsByNewProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, newProfileId); - if (cfExistsByOldProfile || cfExistsByNewProfile) { - sendEntityProfileUpdatedEvent(tenantId, entityId, oldProfileId, newProfileId); - } - } - - private void handleEntityCreate(TenantId tenantId, EntityId entityId, EntityId profileId) { - boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); - if (cfExistsByProfile) { - sendProfileEntityEvent(tenantId, entityId, profileId, true, false); - } - } - @Override public void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId originatorEdgeId) { if (!edgesEnabled) { From 63b79b7242d1c695cbacb46d156495247aa38ba8 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 31 Dec 2024 16:15:42 +0200 Subject: [PATCH 14/14] added completable future to prevent reading old value from while updating is in progress --- .../DefaultCalculatedFieldExecutionService.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 96cbdfc73b..976139e8bd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -94,6 +94,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; @@ -543,13 +544,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); Map argumentsMap = new HashMap<>(argumentValues); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); if (tpi.isMyPartition()) { + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); states.compute(entityCtxId, (ctxId, ctx) -> { CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); + CompletableFuture updateFuture = new CompletableFuture<>(); + Consumer performUpdateState = (state) -> { if (state.updateState(argumentsMap)) { calculatedFieldEntityCtx.setState(state); @@ -561,6 +566,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); } } + updateFuture.complete(null); }; CalculatedFieldState state = calculatedFieldEntityCtx.getState(); @@ -570,7 +576,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); if (!allKeysPresent || requiresTsRollingUpdate) { - Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -581,6 +586,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } else { performUpdateState.accept(state); } + + try { + updateFuture.join(); + } catch (Exception e) { + log.trace("Failed to update state for ctxId [{}].", ctxId, e); + throw new RuntimeException("Failed to update or initialize state.", e); + } + return calculatedFieldEntityCtx; }); } else {