From 621a0984aa85b9e5c9fe293fadee845bb06eec7d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 12 Feb 2025 17:08:56 +0200 Subject: [PATCH 01/11] changed rate limits validation and added validation on state size --- .../server/actors/ActorSystemContext.java | 92 +++++------ ...CalculatedFieldEntityMessageProcessor.java | 12 +- ...alculatedFieldManagerMessageProcessor.java | 3 +- .../CalculatedFieldStateException.java | 24 +++ .../cf/CalculatedFieldStateService.java | 4 +- ...faultCalculatedFieldProcessingService.java | 27 +--- .../service/cf/ctx/state/ArgumentEntry.java | 9 +- .../ctx/state/BaseCalculatedFieldState.java | 17 +- .../cf/ctx/state/CalculatedFieldState.java | 8 +- .../KafkaCalculatedFieldStateService.java | 4 +- .../RocksDBCalculatedFieldStateService.java | 135 ++-------------- .../ctx/state/ScriptCalculatedFieldState.java | 13 -- .../ctx/state/SingleValueArgumentEntry.java | 24 +-- .../cf/ctx/state/TsRollingArgumentEntry.java | 37 +++-- .../server/utils/CalculatedFieldUtils.java | 147 ++++++++++++++++++ .../src/main/resources/thingsboard.yml | 6 - .../state/ScriptCalculatedFieldStateTest.java | 2 +- .../state/SimpleCalculatedFieldStateTest.java | 10 +- .../state/SingleValueArgumentEntryTest.java | 7 +- .../ctx/state/TsRollingArgumentEntryTest.java | 2 +- .../server/common/data/limit/LimitedApi.java | 3 +- .../DefaultTenantProfileConfiguration.java | 1 + common/proto/src/main/proto/queue.proto | 8 +- 23 files changed, 313 insertions(+), 282 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java create mode 100644 application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index c787fc26af..5ac152ca0f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -41,7 +41,9 @@ import org.thingsboard.script.api.js.JsInvokeService; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; +import org.thingsboard.server.cache.limits.RateLimitService; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.LifecycleEvent; @@ -50,6 +52,7 @@ import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.limit.LimitedApi; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.TbActorMsg; @@ -58,6 +61,7 @@ import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.tools.TbRateLimits; +import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.stats.TbApiUsageReportClient; import org.thingsboard.server.dao.alarm.AlarmCommentService; import org.thingsboard.server.dao.asset.AssetProfileService; @@ -107,8 +111,8 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; -import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; import org.thingsboard.server.service.cf.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; @@ -182,8 +186,6 @@ public class ActorSystemContext { private final ConcurrentMap debugPerTenantLimits = new ConcurrentHashMap<>(); - private final ConcurrentMap cfDebugPerTenantLimits = new ConcurrentHashMap<>(); - public ConcurrentMap getDebugPerTenantLimits() { return debugPerTenantLimits; } @@ -452,6 +454,11 @@ public class ActorSystemContext { @Getter private ApiLimitService apiLimitService; + @Lazy + @Autowired() + @Getter + private RateLimitService rateLimitService; + /** * The following Service will be null if we operate in tb-core mode */ @@ -641,14 +648,6 @@ public class ActorSystemContext { @Getter private String deviceStateNodeRateLimitConfig; - @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.enabled:true}") - @Getter - private boolean cfDebugPerTenantEnabled; - - @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") - @Getter - private String cfDebugPerTenantLimitsConfiguration; - @Getter @Setter private TbActorSystem actorSystem; @@ -811,45 +810,40 @@ public class ActorSystemContext { } public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, UUID tbMsgId, TbMsgType tbMsgType, String result, Throwable error) { - if (cfDebugPerTenantEnabled) { - TbRateLimits rateLimits = cfDebugPerTenantLimits.computeIfAbsent(tenantId, id -> new TbRateLimits(cfDebugPerTenantLimitsConfiguration)); - - if (rateLimits.tryConsume()) { - try { - CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() - .tenantId(tenantId) - .entityId(calculatedFieldId.getId()) - .serviceId(getServiceId()) - .calculatedFieldId(calculatedFieldId) - .eventEntity(entityId); - if (tbMsgId != null) { - eventBuilder.msgId(tbMsgId); - } - if (tbMsgType != null) { - eventBuilder.msgType(tbMsgType.name()); - } - if (arguments != null) { - eventBuilder.arguments(JacksonUtil.toString( - arguments.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getValue())) - )); - } - if (result != null) { - eventBuilder.result(result); - } - if (error != null) { - eventBuilder.error(toString(error)); - } - - ListenableFuture future = eventService.saveAsync(eventBuilder.build()); - Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); - } catch (IllegalArgumentException ex) { - log.warn("Failed to persist calculated field debug message", ex); - } - if (log.isTraceEnabled()) { - log.trace("[{}] Tenant level debug mode rate limit detected: {}", tenantId, calculatedFieldId); - } + if (!rateLimitService.checkRateLimit(LimitedApi.CALCULATED_FIELD_DEBUG_EVENTS, tenantId)) { + throw new TbRateLimitsException(EntityType.TENANT); + } + try { + CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() + .tenantId(tenantId) + .entityId(calculatedFieldId.getId()) + .serviceId(getServiceId()) + .calculatedFieldId(calculatedFieldId) + .eventEntity(entityId); + if (tbMsgId != null) { + eventBuilder.msgId(tbMsgId); + } + if (tbMsgType != null) { + eventBuilder.msgType(tbMsgType.name()); } + if (arguments != null) { + eventBuilder.arguments(JacksonUtil.toString( + arguments.entrySet().stream() + .filter(entry -> !entry.getValue().isEmpty()) + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getValue())) + )); + } + if (result != null) { + eventBuilder.result(result); + } + if (error != null) { + eventBuilder.error(toString(error)); + } + + ListenableFuture future = eventService.saveAsync(eventBuilder.build()); + Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); + } catch (IllegalArgumentException ex) { + log.warn("Failed to persist calculated field debug message", ex); } } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 61f7abb297..bfed515eb5 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -38,8 +38,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelem import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldResult; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @@ -225,9 +225,11 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM @SneakyThrows private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); if (state.isReady() && ctx.isInitialized()) { try { CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); + state.checkStateSize(ctxId, ctx.getMaxStateSizeInKBytes()); cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResultMap()), null); @@ -240,7 +242,13 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } else { callback.onSuccess(); // State was updated but no calculation performed; } - cfStateService.persistState(ctx, new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); + try { + cfStateService.persistState(ctxId, state, callback); + } catch (Exception e) { + if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, null, e); + } + } } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 22492f3c4f..898e339426 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -41,9 +41,9 @@ import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; @@ -53,7 +53,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; diff --git a/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java b/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java new file mode 100644 index 0000000000..6248ac1536 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.exception; + +public class CalculatedFieldStateException extends Exception { + + public CalculatedFieldStateException(String message) { + super(message); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java index 37211c66c5..9a41dba576 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java @@ -16,13 +16,13 @@ package org.thingsboard.server.service.cf; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.exception.CalculatedFieldStateException; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; public interface CalculatedFieldStateService { - void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); + void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) throws CalculatedFieldStateException; void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index 6b47053f2d..e0e6db68fe 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -60,8 +60,6 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.usagerecord.ApiLimitService; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; @@ -92,6 +90,7 @@ import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; +import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; @TbRuleEngineComponent @Service @@ -229,19 +228,6 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP return builder.build(); } - //TODO: IM: move to utils; - private CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { - return CalculatedFieldEntityCtxIdProto.newBuilder() - .setTenantIdMSB(ctxId.tenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(ctxId.tenantId().getId().getLeastSignificantBits()) - .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) - .setEntityType(ctxId.entityId().getEntityType().name()) - .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) - .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) - .build(); - } - private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { return switch (argument.getRefEntityKey().getType()) { case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument); @@ -264,7 +250,7 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { return ArgumentEntry.createSingleValueArgument(kvEntry.get()); } else { - return SingleValueArgumentEntry.EMPTY; + return new SingleValueArgumentEntry(); } }, calculatedFieldCallbackExecutor); } @@ -279,7 +265,7 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); - return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); + return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? new TsRollingArgumentEntry(limit, timeWindow) : ArgumentEntry.createTsRollingArgument(tsRolling, limit, timeWindow), calculatedFieldCallbackExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { @@ -311,13 +297,6 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP }; } - private CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { - return CalculatedFieldIdProto.newBuilder() - .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) - .build(); - } - private static class TbCallbackWrapper implements TbQueueCallback { private final TbCallback callback; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index be471a8ad9..6b097124a5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -41,15 +41,14 @@ public interface ArgumentEntry { boolean updateEntry(ArgumentEntry entry); + boolean isEmpty(); + static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { return new SingleValueArgumentEntry(kvEntry); } - static ArgumentEntry createTsRollingArgument(List kvEntries) { - return new TsRollingArgumentEntry(kvEntries); + static ArgumentEntry createTsRollingArgument(List kvEntries, int limit, long timeWindow) { + return new TsRollingArgumentEntry(kvEntries, limit, timeWindow); } - @JsonIgnore - ArgumentEntry copy(); - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index f1ce8038c7..5ea4707ff3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -17,6 +17,8 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.AllArgsConstructor; import lombok.Data; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.utils.CalculatedFieldUtils; import java.util.ArrayList; import java.util.HashMap; @@ -29,6 +31,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { protected List requiredArguments; protected Map arguments; + protected boolean stateTooLarge; public BaseCalculatedFieldState(List requiredArguments) { this.requiredArguments = requiredArguments; @@ -36,7 +39,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { } public BaseCalculatedFieldState() { - this(new ArrayList<>(), new HashMap<>()); + this(new ArrayList<>(), new HashMap<>(), false); } @Override @@ -52,7 +55,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { ArgumentEntry newEntry = entry.getValue(); ArgumentEntry existingEntry = arguments.get(key); - if (existingEntry == null || existingEntry == SingleValueArgumentEntry.EMPTY || existingEntry == TsRollingArgumentEntry.EMPTY) { + if (existingEntry == null) { validateNewEntry(newEntry); arguments.put(key, newEntry); stateUpdated = true; @@ -67,8 +70,14 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { @Override public boolean isReady() { return arguments.keySet().containsAll(requiredArguments) && - !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && - !arguments.containsValue(TsRollingArgumentEntry.EMPTY); + arguments.values().stream().noneMatch(ArgumentEntry::isEmpty); + } + + @Override + public void checkStateSize(CalculatedFieldEntityCtxId ctxId, long maxStateSize) { + if (maxStateSize > 0 && CalculatedFieldUtils.toProto(ctxId, this).getSerializedSize() > maxStateSize) { + setStateTooLarge(true); + } } protected abstract void validateNewEntry(ArgumentEntry newEntry); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 0792b8240d..0219dd2593 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.service.cf.CalculatedFieldResult; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import java.util.List; import java.util.Map; @@ -41,8 +42,6 @@ public interface CalculatedFieldState { Map getArguments(); - List getRequiredArguments(); - void setRequiredArguments(List requiredArguments); boolean updateState(Map argumentValues); @@ -51,4 +50,9 @@ public interface CalculatedFieldState { @JsonIgnore boolean isReady(); + + boolean isStateTooLarge(); + + void checkStateSize(CalculatedFieldEntityCtxId ctxId, long maxStateSize); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java index a1999d438c..f15c5eb416 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -20,8 +20,8 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.queue.util.AfterStartUp; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; @Service @RequiredArgsConstructor @@ -33,7 +33,7 @@ public class KafkaCalculatedFieldStateService implements CalculatedFieldStateSer } @Override - public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + public void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { callback.onSuccess(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java index 3374eb2660..446182e2e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java @@ -20,31 +20,19 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; -import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.util.KvProtoUtil; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; -import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsValueListProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto; +import org.thingsboard.server.exception.CalculatedFieldStateException; import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; import org.thingsboard.server.service.cf.RocksDBService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.CalculatedFieldStateService; import java.util.Map; -import java.util.Optional; -import java.util.TreeMap; -import java.util.UUID; import java.util.stream.Collectors; +import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; +import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; + @Service @RequiredArgsConstructor @ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) // Queue type in mem or Kafka; @@ -67,12 +55,12 @@ public class RocksDBCalculatedFieldStateService implements CalculatedFieldStateS } @Override - public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { - CalculatedFieldStateProto stateProto = toProto(stateId, state); - long maxStateSizeInKBytes = ctx.getMaxStateSizeInKBytes(); - if (maxStateSizeInKBytes <= 0 || stateProto.getSerializedSize() <= maxStateSizeInKBytes) { - rocksDBService.put(toProto(stateId), stateProto); + public void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) throws CalculatedFieldStateException { + if (state.isStateTooLarge()) { + throw new CalculatedFieldStateException("State size exceeds the maximum allowed limit. The state will not be persisted to RocksDB."); } + + rocksDBService.put(toProto(stateId), toProto(stateId, state)); callback.onSuccess(); } @@ -82,107 +70,4 @@ public class RocksDBCalculatedFieldStateService implements CalculatedFieldStateS callback.onSuccess(); } - private CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { - return CalculatedFieldEntityCtxIdProto.newBuilder() - .setTenantIdMSB(ctxId.tenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(ctxId.tenantId().getId().getLeastSignificantBits()) - .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) - .setEntityType(ctxId.entityId().getEntityType().name()) - .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) - .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) - .build(); - } - - private CalculatedFieldEntityCtxId fromProto(CalculatedFieldEntityCtxIdProto ctxIdProto) { - TenantId tenantId = TenantId.fromUUID(new UUID(ctxIdProto.getTenantIdMSB(), ctxIdProto.getTenantIdLSB())); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); - return new CalculatedFieldEntityCtxId(tenantId, calculatedFieldId, entityId); - } - - private CalculatedFieldStateProto toProto(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state) { - CalculatedFieldStateProto.Builder builder = CalculatedFieldStateProto.newBuilder() - .setId(toProto(stateId)) - .setType(state.getType().name()); - - state.getArguments().forEach((argName, argEntry) -> { - if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - builder.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); - } else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) { - builder.addRollingValueArguments(toRollingArgumentProto(argName, rollingArgumentEntry)); - } - }); - - return builder.build(); - } - - private SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { - SingleValueArgumentProto.Builder builder = SingleValueArgumentProto.newBuilder() - .setArgName(argName); - - if (entry != SingleValueArgumentEntry.EMPTY) { - builder.setValue(KvProtoUtil.toTsValueProto(entry.getTs(), entry.getKvEntryValue())); - } - - Optional.ofNullable(entry.getVersion()).ifPresent(builder::setVersion); - - return builder.build(); - } - - private TsValueListProto toRollingArgumentProto(String argName, TsRollingArgumentEntry entry) { - TsValueListProto.Builder builder = TsValueListProto.newBuilder().setKey(argName); - - if (entry != TsRollingArgumentEntry.EMPTY) { - entry.getTsRecords().forEach((ts, value) -> builder.addTsValue(KvProtoUtil.toTsValueProto(ts, value))); - } - - return builder.build(); - } - - private CalculatedFieldState fromProto(CalculatedFieldStateProto proto) { - if (StringUtils.isEmpty(proto.getType())) { - return null; - } - - CalculatedFieldType type = CalculatedFieldType.valueOf(proto.getType()); - - CalculatedFieldState state = switch (type) { - case SIMPLE -> new SimpleCalculatedFieldState(); - case SCRIPT -> new ScriptCalculatedFieldState(); - }; - - proto.getSingleValueArgumentsList().forEach(argProto -> - state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto))); - - if (CalculatedFieldType.SCRIPT.equals(type)) { - proto.getRollingValueArgumentsList().forEach(argProto -> - state.getArguments().put(argProto.getKey(), fromRollingArgumentProto(argProto))); - } - - return state; - } - - private SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) { - if (!proto.hasValue()) { - return (SingleValueArgumentEntry) SingleValueArgumentEntry.EMPTY; - } - TsValueProto tsValueProto = proto.getValue(); - long ts = tsValueProto.getTs(); - BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto); - return new SingleValueArgumentEntry(ts, kvEntry, proto.getVersion()); - } - - private TsRollingArgumentEntry fromRollingArgumentProto(TsValueListProto proto) { - if (proto.getTsValueCount() <= 0) { - return (TsRollingArgumentEntry) TsRollingArgumentEntry.EMPTY; - } - TreeMap tsRecords = new TreeMap<>(); - proto.getTsValueList().forEach(tsValueProto -> { - BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getKey(), tsValueProto); - tsRecords.put(tsValueProto.getTs(), kvEntry); - }); - return new TsRollingArgumentEntry(tsRecords); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 4a24d13c93..0525c5eb90 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -22,14 +22,11 @@ import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.List; import java.util.Map; -import java.util.TreeMap; @Data @Slf4j @@ -51,16 +48,6 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - arguments.forEach((key, argumentEntry) -> { - if (argumentEntry instanceof TsRollingArgumentEntry tsRollingEntry) { - Argument argument = ctx.getArguments().get(key); - TreeMap tsRecords = tsRollingEntry.getTsRecords(); - if (tsRecords.size() > argument.getLimit()) { - tsRecords.pollFirstEntry(); - } - tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); - } - }); Object[] args = ctx.getArgNames().stream() .map(key -> arguments.get(key).getValue()) .toArray(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index d7e5ddd017..f271856ddb 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -32,8 +32,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; @AllArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { - public static final ArgumentEntry EMPTY = new SingleValueArgumentEntry(0); - private long ts; private BasicKvEntry kvEntryValue; private Long version; @@ -61,27 +59,19 @@ public class SingleValueArgumentEntry implements ArgumentEntry { this.kvEntryValue = ProtoUtils.basicKvEntryFromKvEntry(entry); } - /** - * Internal constructor to create immutable SingleValueArgumentEntry.EMPTY - * */ - private SingleValueArgumentEntry(int ignored) { - this.ts = System.currentTimeMillis(); - this.kvEntryValue = null; - } - @Override public ArgumentEntryType getType() { return ArgumentEntryType.SINGLE_VALUE; } - @JsonIgnore - public Object getValue() { - return kvEntryValue.getValue(); + @Override + public boolean isEmpty() { + return kvEntryValue == null; } - @Override - public ArgumentEntry copy() { - return new SingleValueArgumentEntry(this.ts, this.kvEntryValue, this.version); + @JsonIgnore + public Object getValue() { + return isEmpty() ? null : kvEntryValue.getValue(); } @Override @@ -98,7 +88,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { // TODO: should we persist updated ts and version values? BasicKvEntry newValue = singleValueEntry.getKvEntryValue(); - if (this.kvEntryValue.getValue().equals(newValue.getValue())) { + if (this.kvEntryValue != null && this.kvEntryValue.getValue().equals(newValue.getValue())) { return false; } this.kvEntryValue = singleValueEntry.getKvEntryValue(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index ddae0c3513..7b2a743fa9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -37,21 +37,26 @@ import java.util.stream.Collectors; @Slf4j public class TsRollingArgumentEntry implements ArgumentEntry { - public static final ArgumentEntry EMPTY = new TsRollingArgumentEntry(0); - - private static final int MAX_ROLLING_ARGUMENT_ENTRY_SIZE = 1000; - + private Integer limit; + private Long timeWindow; private TreeMap tsRecords = new TreeMap<>(); - public TsRollingArgumentEntry(List kvEntries) { + public TsRollingArgumentEntry(List kvEntries, int limit, long timeWindow) { kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry)); + this.limit = limit; + this.timeWindow = timeWindow; + } + + public TsRollingArgumentEntry(TreeMap tsRecords, int limit, long timeWindow) { + this.tsRecords = tsRecords; + this.limit = limit; + this.timeWindow = timeWindow; } - /** - * Internal constructor to create immutable TsRollingArgumentEntry.EMPTY - */ - private TsRollingArgumentEntry(int ignored) { + public TsRollingArgumentEntry(int limit, long timeWindow) { this.tsRecords = new TreeMap<>(); + this.limit = limit; + this.timeWindow = timeWindow; } @Override @@ -59,6 +64,11 @@ public class TsRollingArgumentEntry implements ArgumentEntry { return ArgumentEntryType.TS_ROLLING; } + @Override + public boolean isEmpty() { + return tsRecords.isEmpty(); + } + @JsonIgnore @Override public Object getValue() { @@ -70,12 +80,6 @@ public class TsRollingArgumentEntry implements ArgumentEntry { (oldValue, newValue) -> oldValue, TreeMap::new )); - - } - - @Override - public ArgumentEntry copy() { - return new TsRollingArgumentEntry(new TreeMap<>(tsRecords)); } @Override @@ -112,9 +116,10 @@ public class TsRollingArgumentEntry implements ArgumentEntry { private void addTsRecord(Long ts, KvEntry value) { if (NumberUtils.isParsable(value.getValue().toString())) { tsRecords.put(ts, ProtoUtils.basicKvEntryFromKvEntry(value)); - if (tsRecords.size() > MAX_ROLLING_ARGUMENT_ENTRY_SIZE) { + if (tsRecords.size() > limit) { tsRecords.pollFirstEntry(); } + tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - timeWindow); } else { throw new IllegalArgumentException("Argument type " + getType() + " only supports numeric values."); } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java new file mode 100644 index 0000000000..d90cc62c8e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -0,0 +1,147 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils; + +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.util.KvProtoUtil; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsRollingArgumentEntryProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsValueListProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; + +import java.util.Optional; +import java.util.TreeMap; +import java.util.UUID; + +public class CalculatedFieldUtils { + + public static CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { + return CalculatedFieldEntityCtxIdProto.newBuilder() + .setTenantIdMSB(ctxId.tenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(ctxId.tenantId().getId().getLeastSignificantBits()) + .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) + .setEntityType(ctxId.entityId().getEntityType().name()) + .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) + .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) + .build(); + } + + public static CalculatedFieldEntityCtxId fromProto(CalculatedFieldEntityCtxIdProto ctxIdProto) { + TenantId tenantId = TenantId.fromUUID(new UUID(ctxIdProto.getTenantIdMSB(), ctxIdProto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); + return new CalculatedFieldEntityCtxId(tenantId, calculatedFieldId, entityId); + } + + public static CalculatedFieldStateProto toProto(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state) { + CalculatedFieldStateProto.Builder builder = CalculatedFieldStateProto.newBuilder() + .setId(toProto(stateId)) + .setType(state.getType().name()); + + state.getArguments().forEach((argName, argEntry) -> { + if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + builder.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); + } else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) { + builder.addRollingValueArguments(toRollingArgumentProto(argName, rollingArgumentEntry)); + } + }); + + return builder.build(); + } + + public static SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { + SingleValueArgumentProto.Builder builder = SingleValueArgumentProto.newBuilder() + .setArgName(argName); + + if (entry.getKvEntryValue() != null) { + builder.setValue(KvProtoUtil.toTsValueProto(entry.getTs(), entry.getKvEntryValue())); + } + + Optional.ofNullable(entry.getVersion()).ifPresent(builder::setVersion); + + return builder.build(); + } + + public static TsRollingArgumentEntryProto toRollingArgumentProto(String argName, TsRollingArgumentEntry entry) { + TsValueListProto.Builder tsRecordsBuilder = TsValueListProto.newBuilder() + .setKey(argName); + + entry.getTsRecords().forEach((ts, value) -> tsRecordsBuilder.addTsValue(KvProtoUtil.toTsValueProto(ts, value))); + + return TsRollingArgumentEntryProto.newBuilder() + .setLimit(entry.getLimit()) + .setTimeWindow(entry.getTimeWindow()) + .setRecords(tsRecordsBuilder.build()) + .build(); + } + + public static CalculatedFieldState fromProto(CalculatedFieldStateProto proto) { + if (StringUtils.isEmpty(proto.getType())) { + return null; + } + + CalculatedFieldType type = CalculatedFieldType.valueOf(proto.getType()); + + CalculatedFieldState state = switch (type) { + case SIMPLE -> new SimpleCalculatedFieldState(); + case SCRIPT -> new ScriptCalculatedFieldState(); + }; + + proto.getSingleValueArgumentsList().forEach(argProto -> + state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto))); + + if (CalculatedFieldType.SCRIPT.equals(type)) { + proto.getRollingValueArgumentsList().forEach(argProto -> + state.getArguments().put(argProto.getRecords().getKey(), fromRollingArgumentProto(argProto))); + } + + return state; + } + + public static SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) { + TsValueProto tsValueProto = proto.getValue(); + long ts = tsValueProto.getTs(); + BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto); + return new SingleValueArgumentEntry(ts, kvEntry, proto.getVersion()); + } + + public static TsRollingArgumentEntry fromRollingArgumentProto(TsRollingArgumentEntryProto proto) { + TsValueListProto records = proto.getRecords(); + String key = records.getKey(); + TreeMap tsRecords = new TreeMap<>(); + records.getTsValueList().forEach(tsValueProto -> { + BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(key, tsValueProto); + tsRecords.put(tsValueProto.getTs(), kvEntry); + }); + return new TsRollingArgumentEntry(tsRecords, proto.getLimit(), proto.getTimeWindow()); + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 197f05a5d5..30a3e51d8c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -510,12 +510,6 @@ actors: js_print_interval_ms: "${ACTORS_JS_STATISTICS_PRINT_INTERVAL_MS:10000}" # Actors statistic persistence frequency in milliseconds persist_frequency: "${ACTORS_STATISTICS_PERSIST_FREQUENCY:3600000}" - calculated_fields: - debug_mode_rate_limits_per_tenant: - # Enable/Disable the rate limit of persisted debug events for all calculated fields per tenant - enabled: "${ACTORS_CF_DEBUG_MODE_RATE_LIMITS_PER_TENANT_ENABLED:true}" - # The value of DEBUG mode rate limit. By default, no more than 50 thousand events per hour - configuration: "${ACTORS_CF_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" debug: settings: diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java index bded0d058d..1429020481 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -189,7 +189,7 @@ public class ScriptCalculatedFieldStateTest { @Test void testIsReadyWhenEmptyEntryPresents() { - state.arguments = new HashMap<>(Map.of("deviceTemperature", TsRollingArgumentEntry.EMPTY, "assetHumidity", assetHumidityArgEntry)); +// state.arguments = new HashMap<>(Map.of("deviceTemperature", TsRollingArgumentEntry.EMPTY, "assetHumidity", assetHumidityArgEntry)); assertThat(state.isReady()).isFalse(); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index d803ad2dab..d87e690478 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -117,10 +117,10 @@ public class SimpleCalculatedFieldStateTest { "key2", key2ArgEntry )); - Map newArgs = Map.of("key3", TsRollingArgumentEntry.EMPTY); - assertThatThrownBy(() -> state.updateState(newArgs)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Rolling argument entry is not supported for simple calculated fields."); +// Map newArgs = Map.of("key3", TsRollingArgumentEntry.EMPTY); +// assertThatThrownBy(() -> state.updateState(newArgs)) +// .isInstanceOf(IllegalArgumentException.class) +// .hasMessage("Rolling argument entry is not supported for simple calculated fields."); } @Test @@ -175,7 +175,7 @@ public class SimpleCalculatedFieldStateTest { "key1", key1ArgEntry, "key2", key2ArgEntry )); - state.getArguments().put("key3", SingleValueArgumentEntry.EMPTY); +// state.getArguments().put("key3", SingleValueArgumentEntry.EMPTY); assertThat(state.isReady()).isFalse(); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java index 13651e852d..57e4655d95 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java @@ -20,7 +20,6 @@ import org.junit.jupiter.api.Test; import org.thingsboard.server.common.data.kv.LongDataEntry; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; public class SingleValueArgumentEntryTest { @@ -40,9 +39,9 @@ public class SingleValueArgumentEntryTest { @Test void testUpdateEntryWhenRollingEntryPassed() { - assertThatThrownBy(() -> entry.updateEntry(TsRollingArgumentEntry.EMPTY)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Unsupported argument entry type for single value argument entry: " + ArgumentEntryType.TS_ROLLING); +// assertThatThrownBy(() -> entry.updateEntry(TsRollingArgumentEntry.EMPTY)) +// .isInstanceOf(IllegalArgumentException.class) +// .hasMessage("Unsupported argument entry type for single value argument entry: " + ArgumentEntryType.TS_ROLLING); } @Test diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java index 5a6f5e96e5..513eec3d8d 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java @@ -40,7 +40,7 @@ public class TsRollingArgumentEntryTest { values.put(ts - 30, new DoubleDataEntry("key", 12.0)); values.put(ts - 20, new DoubleDataEntry("key", 17.0)); - entry = new TsRollingArgumentEntry(values); +// entry = new TsRollingArgumentEntry(values); } @Test diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java b/common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java index 6532a4fe0d..35068821ee 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java @@ -43,7 +43,8 @@ public enum LimitedApi { TRANSPORT_MESSAGES_PER_GATEWAY("transport messages per gateway", false), TRANSPORT_MESSAGES_PER_GATEWAY_DEVICE("transport messages per gateway device", false), EMAILS("emails sending", true), - WS_SUBSCRIPTIONS("WS subscriptions", false); + WS_SUBSCRIPTIONS("WS subscriptions", false), + CALCULATED_FIELD_DEBUG_EVENTS(DefaultTenantProfileConfiguration::getCalculatedFieldDebugEventsRateLimit, "calculated field debug events", true); private Function configExtractor; @Getter diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index 054adbab1b..0139792fa7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -140,6 +140,7 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private long maxDataPointsPerRollingArg; private long maxStateSizeInKBytes; private long maxSingleValueArgumentSizeInKBytes; + private String calculatedFieldDebugEventsRateLimit; @Override public long getProfileThreshold(ApiUsageRecordKey key) { diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 4a2f3710ae..349f507f35 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -814,12 +814,18 @@ message SingleValueArgumentProto { int64 version = 3; } +message TsRollingArgumentEntryProto { + TsValueListProto records = 1; + int32 limit = 2; + int64 timeWindow = 3; +} + message CalculatedFieldStateProto { CalculatedFieldEntityCtxIdProto id = 1; // int32 version = 2; string type = 3; repeated SingleValueArgumentProto singleValueArguments = 4; - repeated TsValueListProto rollingValueArguments = 5; + repeated TsRollingArgumentEntryProto rollingValueArguments = 5; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. From 4245bd4809c6736d562d856fd15c4dbecfa11d22 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 13 Feb 2025 09:56:39 +0200 Subject: [PATCH 02/11] fixed cf debug event filter --- .../common/data/event/CalculatedFieldDebugEventFilter.java | 5 +---- .../dao/sql/event/CalculatedFieldDebugEventRepository.java | 4 ++-- .../thingsboard/server/dao/sql/event/JpaBaseEventDao.java | 6 ++---- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java index 839583ef6c..b458d3dd03 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java @@ -25,8 +25,6 @@ import org.thingsboard.server.common.data.StringUtils; @Schema public class CalculatedFieldDebugEventFilter extends DebugEventFilter { - @Schema(description = "String value representing the calculated field id in the event body", example = "ccbfa2fe-c8f5-45d8-bb37-6b61a6e02833") - protected String calculatedFieldId; @Schema(description = "String value representing the entity id in the event body", example = "57b6bafe-d600-423c-9267-fe31e5218986") protected String entityId; @Schema(description = "String value representing the entity type", allowableValues = "DEVICE") @@ -43,8 +41,7 @@ public class CalculatedFieldDebugEventFilter extends DebugEventFilter { @Override public boolean isNotEmpty() { - return super.isNotEmpty() || !StringUtils.isEmpty(calculatedFieldId) || !StringUtils.isEmpty(entityId) - || !StringUtils.isEmpty(entityType) || !StringUtils.isEmpty(msgId) || !StringUtils.isEmpty(msgType); + return super.isNotEmpty() || !StringUtils.isEmpty(entityId) || !StringUtils.isEmpty(entityType) || !StringUtils.isEmpty(msgId) || !StringUtils.isEmpty(msgType); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java index a759babcfb..dc6f6d7d58 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java @@ -80,7 +80,7 @@ public interface CalculatedFieldDebugEventRepository extends EventRepository findEventByFilter(UUID tenantId, UUID entityId, CalculatedFieldDebugEventFilter eventFilter, TimePageLink pageLink) { - parseUUID(eventFilter.getCalculatedFieldId(), "Calculated Field Id"); parseUUID(eventFilter.getEntityId(), "Entity Id"); parseUUID(eventFilter.getMsgId(), "Message Id"); return DaoUtil.toPageData( @@ -304,7 +303,7 @@ public class JpaBaseEventDao implements EventDao { pageLink.getStartTime(), pageLink.getEndTime(), eventFilter.getServer(), - UUID.fromString(eventFilter.getCalculatedFieldId()), + entityId, eventFilter.getEntityId(), eventFilter.getEntityType(), eventFilter.getMsgId(), @@ -389,7 +388,6 @@ public class JpaBaseEventDao implements EventDao { } private void removeEventsByFilter(UUID tenantId, UUID entityId, CalculatedFieldDebugEventFilter eventFilter, Long startTime, Long endTime) { - parseUUID(eventFilter.getCalculatedFieldId(), "Calculated Field Id"); parseUUID(eventFilter.getEntityId(), "Entity Id"); parseUUID(eventFilter.getMsgId(), "Message Id"); calculatedFieldDebugEventRepository.removeEvents( @@ -398,7 +396,7 @@ public class JpaBaseEventDao implements EventDao { startTime, endTime, eventFilter.getServer(), - UUID.fromString(eventFilter.getCalculatedFieldId()), + entityId, eventFilter.getEntityId(), eventFilter.getEntityType(), eventFilter.getMsgId(), From bd5ecc2e9b2ee4fcf9d4cfbe4a855b88f4fc1a0a Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 13 Feb 2025 11:50:50 +0200 Subject: [PATCH 03/11] added new debug filter property --- .../data/event/CalculatedFieldDebugEventFilter.java | 11 ++++++++++- .../event/CalculatedFieldDebugEventRepository.java | 10 ++++++++++ .../server/dao/sql/event/JpaBaseEventDao.java | 4 ++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java index b458d3dd03..3e56666834 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java @@ -33,6 +33,13 @@ public class CalculatedFieldDebugEventFilter extends DebugEventFilter { protected String msgId; @Schema(description = "String value representing the message type", example = "POST_TELEMETRY_REQUEST") protected String msgType; + @Schema(description = "String value representing the arguments that were used in the calculation performed", + example = "{\"x\":{\"ts\":1739432016629,\"value\":20},\"y\":{\"ts\":1739429717656,\"value\":12}}") + protected String arguments; + @Schema(description = "String value representing the result of a calculation", + example = "{\"x + y\":54}") + protected String result; + @Override public EventType getEventType() { @@ -41,7 +48,9 @@ public class CalculatedFieldDebugEventFilter extends DebugEventFilter { @Override public boolean isNotEmpty() { - return super.isNotEmpty() || !StringUtils.isEmpty(entityId) || !StringUtils.isEmpty(entityType) || !StringUtils.isEmpty(msgId) || !StringUtils.isEmpty(msgType); + return super.isNotEmpty() || !StringUtils.isEmpty(entityId) || !StringUtils.isEmpty(entityType) + || !StringUtils.isEmpty(msgId) || !StringUtils.isEmpty(msgType) + || !StringUtils.isEmpty(arguments) || !StringUtils.isEmpty(result); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java index dc6f6d7d58..56a39c41f2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java @@ -59,6 +59,8 @@ public interface CalculatedFieldDebugEventRepository extends EventRepository Date: Thu, 13 Feb 2025 12:57:26 +0200 Subject: [PATCH 04/11] fixed cf version --- .../server/dao/model/sql/CalculatedFieldEntity.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 64c8e8d5b8..55928dbed1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -31,7 +31,7 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.BaseEntity; -import org.thingsboard.server.dao.model.BaseSqlEntity; +import org.thingsboard.server.dao.model.BaseVersionedEntity; import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; @@ -52,7 +52,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.DEBUG_SETTINGS; @EqualsAndHashCode(callSuper = true) @Entity @Table(name = CALCULATED_FIELD_TABLE_NAME) -public class CalculatedFieldEntity extends BaseSqlEntity implements BaseEntity { +public class CalculatedFieldEntity extends BaseVersionedEntity implements BaseEntity { @Column(name = CALCULATED_FIELD_TENANT_ID_COLUMN) private UUID tenantId; From eae72065ee2dab301efc274e40154ea67302d83d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 13 Feb 2025 13:59:28 +0200 Subject: [PATCH 05/11] fixed integration tests and added debug events creation --- ...CalculatedFieldEntityMessageProcessor.java | 55 ++-- .../CalculatedFieldStateException.java | 2 +- .../cf/ctx/state/TsRollingArgumentEntry.java | 22 +- .../cf/CalculatedFieldIntegrationTest.java | 284 ++++++++++-------- 4 files changed, 198 insertions(+), 165 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index bfed515eb5..66b32dfbfe 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -197,12 +197,18 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM state = getOrInitState(ctx); justRestored = true; } - if (state.updateState(newArgValues) || justRestored) { - cfIdList = new ArrayList<>(cfIdList); - cfIdList.add(ctx.getCfId()); - processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback); - } else { - callback.onSuccess(CALLBACKS_PER_CF); + try { + if (state.updateState(newArgValues) || justRestored) { + cfIdList = new ArrayList<>(cfIdList); + cfIdList.add(ctx.getCfId()); + processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback); + } else { + callback.onSuccess(CALLBACKS_PER_CF); + } + } catch (Exception e) { + if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, null, e); + } } } @@ -212,37 +218,38 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (state != null) { return state; } else { - ListenableFuture stateFuture = systemContext.getCalculatedFieldProcessingService().fetchStateFromDb(ctx, entityId); - // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. - // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. - // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, - // but this will significantly complicate the code. - state = stateFuture.get(1, TimeUnit.MINUTES); - states.put(ctx.getCfId(), state); + try { + ListenableFuture stateFuture = systemContext.getCalculatedFieldProcessingService().fetchStateFromDb(ctx, entityId); + // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. + // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. + // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, + // but this will significantly complicate the code. + state = stateFuture.get(1, TimeUnit.MINUTES); + states.put(ctx.getCfId(), state); + } catch (Exception e) { + if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, null, null, null, null, e); + } + throw new RuntimeException(e); + } } return state; } @SneakyThrows private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); - if (state.isReady() && ctx.isInitialized()) { - try { + try { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); + if (state.isReady() && ctx.isInitialized()) { CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); state.checkStateSize(ctxId, ctx.getMaxStateSizeInKBytes()); cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResultMap()), null); } - } catch (Exception e) { - if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { - systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, null, e); - } + } else { + callback.onSuccess(); // State was updated but no calculation performed; } - } else { - callback.onSuccess(); // State was updated but no calculation performed; - } - try { cfStateService.persistState(ctxId, state, callback); } catch (Exception e) { if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { diff --git a/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java b/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java index 6248ac1536..50ab512cb9 100644 --- a/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java +++ b/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.exception; -public class CalculatedFieldStateException extends Exception { +public class CalculatedFieldStateException extends RuntimeException { public CalculatedFieldStateException(String message) { super(message); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 0834b5583e..cdad1145e8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -25,6 +25,7 @@ import org.thingsboard.script.api.tbel.TbelCfTsDoubleVal; import org.thingsboard.script.api.tbel.TbelCfTsRollingArg; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.exception.CalculatedFieldStateException; import java.util.ArrayList; import java.util.List; @@ -85,7 +86,7 @@ public class TsRollingArgumentEntry implements ArgumentEntry { } @Override - public boolean updateEntry(ArgumentEntry entry) { + public boolean updateEntry(ArgumentEntry entry) throws CalculatedFieldStateException { if (entry instanceof TsRollingArgumentEntry tsRollingEntry) { updateTsRollingEntry(tsRollingEntry); } else if (entry instanceof SingleValueArgumentEntry singleValueEntry) { @@ -107,15 +108,18 @@ public class TsRollingArgumentEntry implements ArgumentEntry { } private void addTsRecord(Long ts, KvEntry value) { - switch (value.getDataType()) { - case LONG -> value.getLongValue().ifPresent(aLong -> tsRecords.put(ts, aLong.doubleValue())); - case DOUBLE -> value.getDoubleValue().ifPresent(aDouble -> tsRecords.put(ts, aDouble)); - case BOOLEAN -> value.getBooleanValue().ifPresent(aBoolean -> tsRecords.put(ts, aBoolean ? 1.0 : 0.0)); - case STRING -> value.getStrValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); - case JSON -> value.getJsonValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); - //TODO: try catch + try { + switch (value.getDataType()) { + case LONG -> value.getLongValue().ifPresent(aLong -> tsRecords.put(ts, aLong.doubleValue())); + case DOUBLE -> value.getDoubleValue().ifPresent(aDouble -> tsRecords.put(ts, aDouble)); + case BOOLEAN -> value.getBooleanValue().ifPresent(aBoolean -> tsRecords.put(ts, aBoolean ? 1.0 : 0.0)); + case STRING -> value.getStrValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); + case JSON -> value.getJsonValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); + } + cleanupExpiredRecords(); + } catch (Exception e) { + throw new IllegalArgumentException("Time series rolling arguments supports only numeric values."); } - cleanupExpiredRecords(); } private void addTsRecord(Long ts, double value) { diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index 91aad7b1cd..f544f463fa 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -40,8 +40,10 @@ import org.thingsboard.server.controller.CalculatedFieldControllerTest; import org.thingsboard.server.dao.service.DaoSqlTest; import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; @DaoSqlTest public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTest { @@ -84,20 +86,22 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // create CF -> perform initial calculation CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); - Thread.sleep(300); - - ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("77.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("77.0"); + }); // update telemetry -> recalculate state doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); - Thread.sleep(300); - - fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + }); // update CF output -> perform calculation with updated output Output savedOutput = savedCalculatedField.getConfiguration().getOutput(); @@ -106,32 +110,35 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes savedOutput.setName("temperatureF"); savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); - Thread.sleep(300); - - ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); - assertThat(temperatureF).isNotNull(); - assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("86.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("86.0"); + }); // update CF argument -> perform calculation with new argument Argument savedArgument = savedCalculatedField.getConfiguration().getArguments().get("T"); savedArgument.setRefEntityKey(new ReferencedEntityKey("deviceTemperature", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); - Thread.sleep(300); - - temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); - assertThat(temperatureF).isNotNull(); - assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); + }); // update CF expression -> perform calculation with new expression savedCalculatedField.getConfiguration().setExpression("1.8 * T + 32"); savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); - Thread.sleep(300); - - temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); - assertThat(temperatureF).isNotNull(); - assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); + }); } @Test @@ -164,20 +171,22 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // create CF -> state is not ready -> no calculation performed CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); - Thread.sleep(300); - - ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + }); // update telemetry -> perform calculation doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); - Thread.sleep(300); - - fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + }); } @Test @@ -211,20 +220,22 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // create CF -> perform initial calculation with default value CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); - Thread.sleep(300); - - ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("53.6"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("53.6"); + }); // update telemetry -> recalculate state doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); - Thread.sleep(300); - - fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + }); } @Test @@ -275,93 +286,100 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // create CF and perform initial calculation doPost("/api/calculatedField", calculatedField, CalculatedField.class); - Thread.sleep(300); - - // result of asset 1 - ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("51.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("51.0"); - // result of asset 2 - ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("52.0"); + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("52.0"); + }); // update device telemetry -> recalculate state for all assets doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":25}")); - Thread.sleep(300); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("36.0"); - // result of asset 1 - z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("36.0"); - - // result of asset 2 - z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + }); // update asset 1 telemetry -> recalculate state only for asset 1 doPost("/api/plugins/telemetry/ASSET/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":15}")); - Thread.sleep(300); - - // result of asset 1 - z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); - // result of asset 2 (no changes) - z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + // result of asset 2 (no changes) + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + }); // update asset 2 telemetry -> recalculate state only for asset 2 doPost("/api/plugins/telemetry/ASSET/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":5}")); - Thread.sleep(300); - - // result of asset 1 (no changes) - z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 (no changes) + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); - // result of asset 2 - z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("30.0"); + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("30.0"); + }); // add new entity to profile -> calculate state for new entity Asset asset3 = createAsset("Test asset 3", assetProfile.getId()); doPost("/api/plugins/telemetry/ASSET/" + asset3.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":13}")); - Thread.sleep(300); - - // result of asset 3 - ArrayNode z3 = getServerAttributes(asset3.getId(), "z"); - assertThat(z3).isNotNull(); - assertThat(z3.get(0).get("value").asText()).isEqualTo("38.0"); + Asset finalAsset3 = asset3; + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 3 + ArrayNode z3 = getServerAttributes(finalAsset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("38.0"); + }); // update device telemetry -> recalculate state for all assets doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":20}")); - Thread.sleep(300); - - // result of asset 1 - z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("35.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("35.0"); - // result of asset 2 - z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("25.0"); + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("25.0"); - // result of asset 3 - z3 = getServerAttributes(asset3.getId(), "z"); - assertThat(z3).isNotNull(); - assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + // result of asset 3 + ArrayNode z3 = getServerAttributes(finalAsset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + }); // update profile for asset 3 -> delete state for asset 3 AssetProfile newAssetProfile = doPost("/api/assetProfile", createAssetProfile("New Asset Profile"), AssetProfile.class); @@ -371,22 +389,24 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // update device telemetry -> recalculate state for asset 1 and asset 2 doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":15}")); - Thread.sleep(300); - - // result of asset 1 - z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("30.0"); - - // result of asset 2 - z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("20.0"); - - // no changes for asset 3 - z3 = getServerAttributes(asset3.getId(), "z"); - assertThat(z3).isNotNull(); - assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + Asset updatedAsset3 = asset3; + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("30.0"); + + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("20.0"); + + // no changes for asset 3 + ArrayNode z3 = getServerAttributes(updatedAsset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + }); } @Test @@ -421,20 +441,22 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // create CF -> ctx is not initialized -> no calculation perform CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); - Thread.sleep(300); - - ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + }); // update telemetry -> ctx is not initialized -> no calculation perform doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); - Thread.sleep(300); - - fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + }); } private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { From 7911384f375f8e823afd51974a53315598673aa4 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 14 Feb 2025 16:42:55 +0200 Subject: [PATCH 06/11] fixed tests, added error without stacktrace and param to system params --- .../server/actors/ActorSystemContext.java | 74 +++++++++++-------- ...alculatedFieldManagerMessageProcessor.java | 14 ++-- .../controller/SystemInfoController.java | 11 ++- .../cf/ctx/state/TsRollingArgumentEntry.java | 2 +- .../src/main/resources/thingsboard.yml | 6 ++ .../state/ScriptCalculatedFieldStateTest.java | 55 ++------------ .../state/SimpleCalculatedFieldStateTest.java | 10 +-- .../state/SingleValueArgumentEntryTest.java | 7 +- .../ctx/state/TsRollingArgumentEntryTest.java | 60 +++++++++++---- .../server/common/data/SystemParams.java | 1 + .../data/event/CalculatedFieldDebugEvent.java | 1 - 11 files changed, 125 insertions(+), 116 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index b8344c6c0a..20f61115e6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -43,7 +43,6 @@ import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.cache.limits.RateLimitService; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.LifecycleEvent; @@ -624,6 +623,14 @@ public class ActorSystemContext { @Getter private String debugPerTenantLimitsConfiguration; + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.enabled:true}") + @Getter + private boolean calculatedFieldsDebugPerTenantEnabled; + + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") + @Getter + private String calculatedFieldsDebugPerTenantLimitsConfiguration; + @Value("${actors.rpc.submit_strategy:BURST}") @Getter private String rpcSubmitStrategy; @@ -810,39 +817,42 @@ public class ActorSystemContext { } public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, UUID tbMsgId, TbMsgType tbMsgType, String result, Throwable error) { - if (!rateLimitService.checkRateLimit(LimitedApi.CALCULATED_FIELD_DEBUG_EVENTS, tenantId)) { - throw new TbRateLimitsException(EntityType.TENANT); - } - try { - CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() - .tenantId(tenantId) - .entityId(calculatedFieldId.getId()) - .serviceId(getServiceId()) - .calculatedFieldId(calculatedFieldId) - .eventEntity(entityId); - if (tbMsgId != null) { - eventBuilder.msgId(tbMsgId); - } - if (tbMsgType != null) { - eventBuilder.msgType(tbMsgType.name()); - } - if (arguments != null) { - eventBuilder.arguments(JacksonUtil.toString( - arguments.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toTbelCfArg())) - )); - } - if (result != null) { - eventBuilder.result(result); - } - if (error != null) { - eventBuilder.error(toString(error)); + if (calculatedFieldsDebugPerTenantEnabled) { + if (!rateLimitService.checkRateLimit(LimitedApi.CALCULATED_FIELD_DEBUG_EVENTS, (Object) tenantId, calculatedFieldsDebugPerTenantLimitsConfiguration)) { + log.trace("[{}] Calculated field debug event limits exceeded!", tenantId); + throw new TbRateLimitsException("Failed to persist calculated field debug event due to rate limits!"); } + try { + CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() + .tenantId(tenantId) + .entityId(calculatedFieldId.getId()) + .serviceId(getServiceId()) + .calculatedFieldId(calculatedFieldId) + .eventEntity(entityId); + if (tbMsgId != null) { + eventBuilder.msgId(tbMsgId); + } + if (tbMsgType != null) { + eventBuilder.msgType(tbMsgType.name()); + } + if (arguments != null) { + eventBuilder.arguments(JacksonUtil.toString( + arguments.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toTbelCfArg())) + )); + } + if (result != null) { + eventBuilder.result(result); + } + if (error != null) { + eventBuilder.error(error.getMessage()); + } - ListenableFuture future = eventService.saveAsync(eventBuilder.build()); - Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); - } catch (IllegalArgumentException ex) { - log.warn("Failed to persist calculated field debug message", ex); + ListenableFuture future = eventService.saveAsync(eventBuilder.build()); + Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); + } catch (IllegalArgumentException ex) { + log.warn("Failed to persist calculated field debug message", ex); + } } } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 898e339426..c3aa378524 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -262,6 +262,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware callback.onSuccess(); } else { var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); + try { + newCfCtx.init(); + } catch (Exception e) { + if (DebugModeUtil.isDebugAllAvailable(newCf)) { + systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e); + } + } calculatedFields.put(newCf.getId(), newCfCtx); List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); List newCfList = new ArrayList<>(oldCfList.size()); @@ -286,13 +293,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) var stateChanges = newCfCtx.hasStateChanges(oldCfCtx); if (stateChanges || newCfCtx.hasOtherSignificantChanges(oldCfCtx)) { - try { - newCfCtx.init(); - } catch (Exception e) { - if (DebugModeUtil.isDebugAllAvailable(newCf)) { - systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e); - } - } initCf(newCfCtx, callback, stateChanges); } else { callback.onSuccess(); diff --git a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java index 8b958df68b..d630c5a22a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java +++ b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java @@ -35,8 +35,8 @@ import org.thingsboard.server.common.data.SystemParams; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.mobile.qrCodeSettings.QrCodeSettings; import org.thingsboard.server.common.data.mobile.qrCodeSettings.QRCodeConfig; +import org.thingsboard.server.common.data.mobile.qrCodeSettings.QrCodeSettings; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.settings.UserSettings; import org.thingsboard.server.common.data.settings.UserSettingsType; @@ -80,6 +80,12 @@ public class SystemInfoController extends BaseController { @Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") private String ruleChainDebugPerTenantLimitsConfiguration; + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.enabled:true}") + private boolean calculatedFieldDebugPerTenantLimitsEnabled; + + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") + private String calculatedFieldDebugPerTenantLimitsConfiguration; + @Autowired(required = false) private BuildProperties buildProperties; @@ -155,6 +161,9 @@ public class SystemInfoController extends BaseController { if (ruleChainDebugPerTenantLimitsEnabled) { systemParams.setRuleChainDebugPerTenantLimitsConfiguration(ruleChainDebugPerTenantLimitsConfiguration); } + if (calculatedFieldDebugPerTenantLimitsEnabled) { + systemParams.setCalculatedFieldDebugPerTenantLimitsConfiguration(calculatedFieldDebugPerTenantLimitsConfiguration); + } } systemParams.setMobileQrEnabled(Optional.ofNullable(qrCodeSettingService.findQrCodeSettings(TenantId.SYS_TENANT_ID)) .map(QrCodeSettings::getQrCodeConfig).map(QRCodeConfig::isShowOnHomePage) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index cdad1145e8..1ecf38746e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -43,9 +43,9 @@ public class TsRollingArgumentEntry implements ArgumentEntry { private TreeMap tsRecords = new TreeMap<>(); public TsRollingArgumentEntry(List kvEntries, int limit, long timeWindow) { - kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry)); this.limit = limit; this.timeWindow = timeWindow; + kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry)); } public TsRollingArgumentEntry(TreeMap tsRecords, int limit, long timeWindow) { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 30a3e51d8c..35e8c0e970 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -510,6 +510,12 @@ actors: js_print_interval_ms: "${ACTORS_JS_STATISTICS_PRINT_INTERVAL_MS:10000}" # Actors statistic persistence frequency in milliseconds persist_frequency: "${ACTORS_STATISTICS_PERSIST_FREQUENCY:3600000}" + calculated_fields: + debug_mode_rate_limits_per_tenant: + # Enable/Disable the rate limit of persisted debug events for all calculated fields per tenant + enabled: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_ENABLED:true}" + # The value of DEBUG mode rate limit. By default, no more than 50 thousand events per hour + configuration: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" debug: settings: diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java index 30ad7a6507..0227365ed7 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -35,7 +35,7 @@ import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedField import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -57,7 +57,7 @@ public class ScriptCalculatedFieldStateTest { private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); - private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new LongDataEntry("assetHumidity", 43L), 122L); + private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new DoubleDataEntry("assetHumidity", 43.0), 122L); private final TsRollingArgumentEntry deviceTemperatureArgEntry = createRollingArgEntry(); private final long ts = System.currentTimeMillis(); @@ -127,52 +127,7 @@ public class ScriptCalculatedFieldStateTest { Output output = getCalculatedFieldConfig().getOutput(); assertThat(result.getType()).isEqualTo(output.getType()); assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 13.0, "assetHumidity", 43L)); - } - - @Test - void testPerformCalculationWhenOldTelemetry() throws ExecutionException, InterruptedException { - TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); - - TreeMap values = new TreeMap<>(); - values.put(ts - 40000, 4.0);// will not be used for calculation - values.put(ts - 45000, 2.0);// will not be used for calculation - values.put(ts - 20, 0.0); - - argumentEntry.setTsRecords(values); - - state.arguments = new HashMap<>(Map.of("deviceTemperature", argumentEntry, "assetHumidity", assetHumidityArgEntry)); - - CalculatedFieldResult result = state.performCalculation(ctx).get(); - - assertThat(result).isNotNull(); - Output output = getCalculatedFieldConfig().getOutput(); - assertThat(result.getType()).isEqualTo(output.getType()); - assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43L)); - } - - @Test - void testPerformCalculationWhenArgumentsMoreThanLimit() throws ExecutionException, InterruptedException { - TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); - TreeMap values = new TreeMap<>(); - values.put(ts - 20, 1000.0);// will not be used - values.put(ts - 18, 0.0); - values.put(ts - 16, 0.0); - values.put(ts - 14, 0.0); - values.put(ts - 12, 0.0); - values.put(ts - 10, 0.0); - argumentEntry.setTsRecords(values); - - state.arguments = new HashMap<>(Map.of("deviceTemperature", argumentEntry, "assetHumidity", assetHumidityArgEntry)); - - CalculatedFieldResult result = state.performCalculation(ctx).get(); - - assertThat(result).isNotNull(); - Output output = getCalculatedFieldConfig().getOutput(); - assertThat(result.getType()).isEqualTo(output.getType()); - assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43L)); + assertThat(result.getResultMap()).isEqualTo(Map.of("maxDeviceTemperature", 17.0, "assetHumidity", 43.0)); } @Test @@ -189,7 +144,7 @@ public class ScriptCalculatedFieldStateTest { @Test void testIsReadyWhenEmptyEntryPresents() { -// state.arguments = new HashMap<>(Map.of("deviceTemperature", TsRollingArgumentEntry.EMPTY, "assetHumidity", assetHumidityArgEntry)); + state.arguments = new HashMap<>(Map.of("deviceTemperature", new TsRollingArgumentEntry(5, 30000L), "assetHumidity", assetHumidityArgEntry)); assertThat(state.isReady()).isFalse(); } @@ -235,7 +190,7 @@ public class ScriptCalculatedFieldStateTest { config.setArguments(Map.of("deviceTemperature", argument1, "assetHumidity", argument2)); - config.setExpression("var result = 0; foreach(element : deviceTemperature.entrySet()) { result += element.getValue(); } var map = {}; map.put(\"averageDeviceTemperature\", result / deviceTemperature.size()); map.put(\"assetHumidity\", assetHumidity); return map;"); + config.setExpression("return {\"maxDeviceTemperature\": deviceTemperature.max(), \"assetHumidity\": assetHumidity.value}"); Output output = new Output(); output.setType(OutputType.ATTRIBUTES); diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index d87e690478..c451ffcbfc 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -117,10 +117,10 @@ public class SimpleCalculatedFieldStateTest { "key2", key2ArgEntry )); -// Map newArgs = Map.of("key3", TsRollingArgumentEntry.EMPTY); -// assertThatThrownBy(() -> state.updateState(newArgs)) -// .isInstanceOf(IllegalArgumentException.class) -// .hasMessage("Rolling argument entry is not supported for simple calculated fields."); + Map newArgs = Map.of("key3", new TsRollingArgumentEntry(10, 30000L)); + assertThatThrownBy(() -> state.updateState(newArgs)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Rolling argument entry is not supported for simple calculated fields."); } @Test @@ -175,7 +175,7 @@ public class SimpleCalculatedFieldStateTest { "key1", key1ArgEntry, "key2", key2ArgEntry )); -// state.getArguments().put("key3", SingleValueArgumentEntry.EMPTY); + state.getArguments().put("key3", new SingleValueArgumentEntry()); assertThat(state.isReady()).isFalse(); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java index 57e4655d95..2514b9f34e 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java @@ -20,6 +20,7 @@ import org.junit.jupiter.api.Test; import org.thingsboard.server.common.data.kv.LongDataEntry; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class SingleValueArgumentEntryTest { @@ -39,9 +40,9 @@ public class SingleValueArgumentEntryTest { @Test void testUpdateEntryWhenRollingEntryPassed() { -// assertThatThrownBy(() -> entry.updateEntry(TsRollingArgumentEntry.EMPTY)) -// .isInstanceOf(IllegalArgumentException.class) -// .hasMessage("Unsupported argument entry type for single value argument entry: " + ArgumentEntryType.TS_ROLLING); + assertThatThrownBy(() -> entry.updateEntry(new TsRollingArgumentEntry(5, 30000L))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported argument entry type for single value argument entry: " + ArgumentEntryType.TS_ROLLING); } @Test diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java index e4d5a59d03..75aca61825 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf.ctx.state; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; @@ -40,7 +39,7 @@ public class TsRollingArgumentEntryTest { values.put(ts - 30, 12.0); values.put(ts - 20, 17.0); -// entry = new TsRollingArgumentEntry(values); + entry = new TsRollingArgumentEntry(5, 30000L, values); } @Test @@ -57,18 +56,10 @@ public class TsRollingArgumentEntryTest { assertThat(entry.getTsRecords().get(ts - 10)).isEqualTo(23.0); } - @Test - void testUpdateEntryWhenSingleValueEntryWithTheSameTsPassed() { - SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 20, new DoubleDataEntry("key", 23.0), 123L); - - assertThat(entry.updateEntry(newEntry)).isFalse(); - } - @Test void testUpdateEntryWhenRollingEntryPassed() { TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); TreeMap values = new TreeMap<>(); - values.put(ts - 20, 16.0); values.put(ts - 10, 7.0); values.put(ts - 5, 1.0); newEntry.setTsRecords(values); @@ -76,11 +67,11 @@ public class TsRollingArgumentEntryTest { assertThat(entry.updateEntry(newEntry)).isTrue(); assertThat(entry.getTsRecords()).hasSize(5); assertThat(entry.getTsRecords()).isEqualTo(Map.of( - ts - 40, new DoubleDataEntry("key", 10.0), - ts - 30, new DoubleDataEntry("key", 12.0), - ts - 20, new DoubleDataEntry("key", 17.0), - ts - 10, new DoubleDataEntry("key", 7.0), - ts - 5, new DoubleDataEntry("key", 1.0) + ts - 40, 10.0, + ts - 30, 12.0, + ts - 20, 17.0, + ts - 10, 7.0, + ts - 5, 1.0 )); } @@ -90,7 +81,44 @@ public class TsRollingArgumentEntryTest { assertThatThrownBy(() -> entry.updateEntry(newEntry)) .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Argument type " + ArgumentEntryType.TS_ROLLING + " only supports numeric values."); + .hasMessage("Time series rolling arguments supports only numeric values."); + } + + @Test + void testUpdateEntryWhenOldTelemetry() { + TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); + TreeMap values = new TreeMap<>(); + values.put(ts - 40000, 4.0);// will not be used for calculation + values.put(ts - 45000, 2.0);// will not be used for calculation + values.put(ts - 5, 0.0); + newEntry.setTsRecords(values); + + entry = new TsRollingArgumentEntry(3, 30000L); + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords()).hasSize(1); + assertThat(entry.getTsRecords()).isEqualTo(Map.of( + ts - 5, 0.0 + )); + } + + @Test + void testPerformCalculationWhenArgumentsMoreThanLimit() { + TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); + TreeMap values = new TreeMap<>(); + values.put(ts - 20, 1000.0);// will not be used + values.put(ts - 18, 0.0); + values.put(ts - 16, 0.0); + values.put(ts - 14, 0.0); + newEntry.setTsRecords(values); + + entry = new TsRollingArgumentEntry(3, 30000L); + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords()).hasSize(3); + assertThat(entry.getTsRecords()).isEqualTo(Map.of( + ts - 18, 0.0, + ts - 16, 0.0, + ts - 14, 0.0 + )); } } \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java index abe1932327..53f39977c1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java @@ -34,4 +34,5 @@ public class SystemParams { boolean mobileQrEnabled; int maxDebugModeDurationMinutes; String ruleChainDebugPerTenantLimitsConfiguration; + String calculatedFieldDebugPerTenantLimitsConfiguration; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java index e0599db358..9b64dd0d40 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java @@ -91,5 +91,4 @@ public class CalculatedFieldDebugEvent extends Event { return eventInfo; } - } From 7642a884609df53a1f962f35ce70cf999ae263b0 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 17 Feb 2025 10:02:26 +0200 Subject: [PATCH 07/11] changed calculation result --- .../CalculatedFieldEntityMessageProcessor.java | 2 +- .../controller/CalculatedFieldController.java | 14 +++++++++----- .../server/service/cf/CalculatedFieldResult.java | 10 ++-------- .../DefaultCalculatedFieldProcessingService.java | 12 +----------- .../cf/ctx/state/CalculatedFieldScriptEngine.java | 6 ------ .../state/CalculatedFieldTbelScriptEngine.java | 15 --------------- .../cf/ctx/state/ScriptCalculatedFieldState.java | 4 ++-- .../cf/ctx/state/SimpleCalculatedFieldState.java | 3 ++- .../ctx/state/ScriptCalculatedFieldStateTest.java | 3 ++- .../ctx/state/SimpleCalculatedFieldStateTest.java | 3 ++- 10 files changed, 21 insertions(+), 51 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 66b32dfbfe..60c76b720d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -245,7 +245,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM state.checkStateSize(ctxId, ctx.getMaxStateSizeInKBytes()); cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { - systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResultMap()), null); + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null); } } else { callback.onSuccess(); // State was updated but no calculation performed; diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 5c8b40dcad..33998801ff 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -84,14 +84,18 @@ public class CalculatedFieldController extends BaseController { private static final String TEST_SCRIPT_EXPRESSION = "Execute the Script expression and return the result. The format of request: \n\n" + MARKDOWN_CODE_BLOCK_START + "{\n" + - " \"expression\": \"var temp = 0; foreach(element: temperature.entrySet()) { temp += element.getValue(); } var avgTemperature = temp / temperature.size(); var adjustedTemperature = avgTemperature + 0.1 * humidity; return { \\\"adjustedTemperature\\\": adjustedTemperature };\",\n" + + " \"expression\": \"var temp = 0; foreach(element: temperature.values) {temp += element.value;} var avgTemperature = temp / temperature.values.size(); var adjustedTemperature = avgTemperature + 0.1 * humidity.value; return {\"adjustedTemperature\": adjustedTemperature};\",\n" + " \"arguments\": {\n" + " \"temperature\": {\n" + - " \"14327856345\": 22.4,\n" + - " \"14327857298\": 21.9,\n" + - " \"14327857510\": 22.0\n" + + " \"values\": [\n" + + " { \"ts\": 1739775639851, \"value\": 23 },\n" + + " { \"ts\": 1739775664561, \"value\": 43 },\n" + + " { \"ts\": 1739775713079, \"value\": 15 },\n" + + " { \"ts\": 1739775999522, \"value\": 34 },\n" + + " { \"ts\": 1739776228452, \"value\": 22 }\n" + + " ]\n" + " },\n" + - " \"humidity\": 42\n" + + " \"humidity\": { \"ts\": 1739776478057, \"value\": 23 }\n" + " }\n" + "}" + MARKDOWN_CODE_BLOCK_END diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index 52e0a0151b..13ac4972e2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -15,22 +15,16 @@ */ package org.thingsboard.server.service.cf; +import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.configuration.OutputType; -import java.util.Map; - @Data public final class CalculatedFieldResult { private final OutputType type; private final AttributeScope scope; - private final Map resultMap; + private final JsonNode result; - public CalculatedFieldResult(OutputType type, AttributeScope scope, Map resultMap) { - this.type = type; - this.scope = scope; - this.resultMap = resultMap == null ? Map.of() : Map.copyOf(resultMap); - } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index e0e6db68fe..aeb31b3393 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -15,8 +15,6 @@ */ package org.thingsboard.server.service.cf; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -155,8 +153,7 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP OutputType type = calculatedFieldResult.getType(); TbMsgType msgType = OutputType.ATTRIBUTES.equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; TbMsgMetaData md = OutputType.ATTRIBUTES.equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; - ObjectNode payload = createJsonPayload(calculatedFieldResult); - TbMsg msg = TbMsg.newMsg().type(msgType).originator(entityId).previousCalculatedFieldIds(cfIds).metaData(md).data(JacksonUtil.writeValueAsString(payload)).build(); + TbMsg msg = TbMsg.newMsg().type(msgType).originator(entityId).previousCalculatedFieldIds(cfIds).metaData(md).data(JacksonUtil.writeValueAsString(calculatedFieldResult.getResult())).build(); clusterService.pushMsgToRuleEngine(tenantId, entityId, msg, new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { @@ -283,13 +280,6 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP return new StringDataEntry(key, defaultValue); } - private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { - ObjectNode payload = JacksonUtil.newObjectNode(); - Map resultMap = calculatedFieldResult.getResultMap(); - resultMap.forEach((k, v) -> payload.set(k, JacksonUtil.convertValue(v, JsonNode.class))); - return payload; - } - private CalculatedFieldState createStateByType(CalculatedFieldCtx ctx) { return switch (ctx.getCfType()) { case SIMPLE -> new SimpleCalculatedFieldState(ctx.getArgNames()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java index 6bea6ce705..b54ef744a0 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java @@ -18,18 +18,12 @@ package org.thingsboard.server.service.cf.ctx.state; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.ListenableFuture; -import java.util.Map; - public interface CalculatedFieldScriptEngine { ListenableFuture executeScriptAsync(Object[] args); - ListenableFuture> executeToMapAsync(Object[] args); - ListenableFuture executeJsonAsync(Object[] args); - ListenableFuture> executeToMapTransform(Object result); - void destroy(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java index 6dcbf33067..c298f9cbe3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; -import io.jsonwebtoken.lang.Collections; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.ScriptType; @@ -27,7 +26,6 @@ import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.id.TenantId; import javax.script.ScriptException; -import java.util.Map; import java.util.UUID; import java.util.concurrent.ExecutionException; @@ -72,24 +70,11 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng }, MoreExecutors.directExecutor()); } - @Override - public ListenableFuture> executeToMapAsync(Object[] args) { - return Futures.transformAsync(executeScriptAsync(args), this::executeToMapTransform, MoreExecutors.directExecutor()); - } - @Override public ListenableFuture executeJsonAsync(Object[] args) { return Futures.transform(executeScriptAsync(args), JacksonUtil::valueToTree, MoreExecutors.directExecutor()); } - @Override - public ListenableFuture> executeToMapTransform(Object result) { - if (result instanceof Map) { - return Futures.immediateFuture((Map) result); - } - throw new IllegalArgumentException("Wrong result type: [" + result.getClass().getName() + "]"); - } - @Override public void destroy() { tbelInvokeService.release(this.scriptId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index df4cab43ef..94f0894ff3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; @@ -27,7 +28,6 @@ import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.List; -import java.util.Map; @Data @Slf4j @@ -53,7 +53,7 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { .map(this::toTbelArgument) .toArray(); - ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); + ListenableFuture resultFuture = ctx.getCalculatedFieldScriptEngine().executeJsonAsync(args); Output output = ctx.getOutput(); return Futures.transform(resultFuture, result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 01cf9cff70..fa76a63e25 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; import lombok.NoArgsConstructor; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.kv.BasicKvEntry; @@ -63,7 +64,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { double expressionResult = expr.evaluate(); Output output = ctx.getOutput(); - return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), Map.of(output.getName(), expressionResult))); + return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), JacksonUtil.valueToTree(Map.of(output.getName(), expressionResult)))); } } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java index 0227365ed7..bc75bd74ff 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -20,6 +20,7 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.tbel.DefaultTbelInvokeService; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.AttributeScope; @@ -127,7 +128,7 @@ public class ScriptCalculatedFieldStateTest { Output output = getCalculatedFieldConfig().getOutput(); assertThat(result.getType()).isEqualTo(output.getType()); assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("maxDeviceTemperature", 17.0, "assetHumidity", 43.0)); + assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("maxDeviceTemperature", 17.0, "assetHumidity", 43.0))); } @Test diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index c451ffcbfc..55e65422eb 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -20,6 +20,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -137,7 +138,7 @@ public class SimpleCalculatedFieldStateTest { Output output = getCalculatedFieldConfig().getOutput(); assertThat(result.getType()).isEqualTo(output.getType()); assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("output", 49.0)); + assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 49.0))); } @Test From 1e46100cd2c161af8640db9a29dab5637fa7b17a Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 17 Feb 2025 11:51:30 +0200 Subject: [PATCH 08/11] added endTs and startTs to rolling argument --- .../service/cf/ctx/state/TsRollingArgumentEntry.java | 3 ++- .../thingsboard/script/api/tbel/TbelCfTsRollingArg.java | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 1ecf38746e..92722fb96b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -82,7 +82,8 @@ public class TsRollingArgumentEntry implements ArgumentEntry { for (var e : tsRecords.entrySet()) { values.add(new TbelCfTsDoubleVal(e.getKey(), e.getValue())); } - return new TbelCfTsRollingArg(values); + long ts = System.currentTimeMillis(); + return new TbelCfTsRollingArg(ts - timeWindow, ts, values); } @Override diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java index 14122bbde8..230baaee0a 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java @@ -27,10 +27,16 @@ import static org.thingsboard.script.api.tbel.TbelCfTsDoubleVal.OBJ_SIZE; public class TbelCfTsRollingArg implements TbelCfArg, Iterable { + @Getter + private final long startTs; + @Getter + private final long endTs; @Getter private final List values; - public TbelCfTsRollingArg(List values) { + public TbelCfTsRollingArg(long startTs, long endTs, List values) { + this.startTs = startTs; + this.endTs = endTs; this.values = Collections.unmodifiableList(values); } From 5e9ee15b1df33525dbec21c7fad26c892caffedd Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 17 Feb 2025 14:55:04 +0200 Subject: [PATCH 09/11] fixed proto utils test --- .../thingsboard/server/common/util/ProtoUtils.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 07406977d2..cbcbba4fc7 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ResourceSubType; import org.thingsboard.server.common.data.ResourceType; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; @@ -148,9 +149,13 @@ public class ProtoUtils { var builder = ComponentLifecycleMsg.builder() .tenantId(TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB()))) .entityId(entityId) - .event(ComponentLifecycleEvent.values()[proto.getEventValue()]) - .name(proto.getName()) - .oldName(proto.getOldName()); + .event(ComponentLifecycleEvent.values()[proto.getEventValue()]); + if (!StringUtils.isEmpty(proto.getName())) { + builder.name(proto.getName()); + } + if (!StringUtils.isEmpty(proto.getOldName())) { + builder.oldName(proto.getOldName()); + } if (proto.getProfileIdMSB() != 0 || proto.getProfileIdLSB() != 0) { var profileType = EntityType.DEVICE.equals(entityId.getEntityType()) ? EntityType.DEVICE_PROFILE : EntityType.ASSET_PROFILE; builder.profileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB()))); From baf0e948dbc2f73ddb0ce3e287e53c8167690942 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 17 Feb 2025 17:21:12 +0200 Subject: [PATCH 10/11] replaced cf debug events creation --- .../server/actors/ActorSystemContext.java | 16 ++- ...CalculatedFieldEntityMessageProcessor.java | 124 +++++++++--------- .../cf/ctx/state/TsRollingArgumentEntry.java | 3 +- 3 files changed, 75 insertions(+), 68 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 20f61115e6..14c8a41fb5 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -60,7 +60,6 @@ import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.tools.TbRateLimits; -import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.stats.TbApiUsageReportClient; import org.thingsboard.server.dao.alarm.AlarmCommentService; import org.thingsboard.server.dao.asset.AssetProfileService; @@ -817,11 +816,7 @@ public class ActorSystemContext { } public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, UUID tbMsgId, TbMsgType tbMsgType, String result, Throwable error) { - if (calculatedFieldsDebugPerTenantEnabled) { - if (!rateLimitService.checkRateLimit(LimitedApi.CALCULATED_FIELD_DEBUG_EVENTS, (Object) tenantId, calculatedFieldsDebugPerTenantLimitsConfiguration)) { - log.trace("[{}] Calculated field debug event limits exceeded!", tenantId); - throw new TbRateLimitsException("Failed to persist calculated field debug event due to rate limits!"); - } + if (checkLimits(tenantId)) { try { CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() .tenantId(tenantId) @@ -856,6 +851,15 @@ public class ActorSystemContext { } } + private boolean checkLimits(TenantId tenantId) { + if (calculatedFieldsDebugPerTenantEnabled && + !rateLimitService.checkRateLimit(LimitedApi.CALCULATED_FIELD_DEBUG_EVENTS, (Object) tenantId, calculatedFieldsDebugPerTenantLimitsConfiguration)) { + log.trace("[{}] Calculated field debug event limits exceeded!", tenantId); + return false; + } + return true; + } + public static Exception toException(Throwable error) { return Exception.class.isInstance(error) ? (Exception) error : new Exception(error); } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 60c76b720d..17ec258c80 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -107,8 +107,15 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM log.info("Force reinitialization of CF: [{}].", cfCtx.getCfId()); states.remove(cfCtx.getCfId()); } - var cfState = getOrInitState(cfCtx); - processStateIfReady(cfCtx, Collections.singletonList(cfCtx.getCfId()), cfState, null, null, msg.getCallback()); + try { + var cfState = getOrInitState(cfCtx); + processStateIfReady(cfCtx, Collections.singletonList(cfCtx.getCfId()), cfState, null, null, msg.getCallback()); + } catch (Exception e) { + if (DebugModeUtil.isDebugFailuresAvailable(cfCtx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, cfCtx.getCfId(), entityId, null, null, null, null, e); + } + msg.getCallback().onFailure(e); + } } public void process(CalculatedFieldEntityDeleteMsg msg) { @@ -146,31 +153,45 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM var proto = msg.getProto(); var ctx = msg.getCtx(); var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); - List cfIds = getCalculatedFieldIds(proto); - if (cfIds.contains(ctx.getCfId())) { - callback.onSuccess(CALLBACKS_PER_CF); - } else { - if (proto.getTsDataCount() > 0) { - processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); - } else if (proto.getAttrDataCount() > 0) { - processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); - } else { + try { + List cfIds = getCalculatedFieldIds(proto); + if (cfIds.contains(ctx.getCfId())) { callback.onSuccess(CALLBACKS_PER_CF); + } else { + if (proto.getTsDataCount() > 0) { + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); + } else if (proto.getAttrDataCount() > 0) { + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); + } else { + callback.onSuccess(CALLBACKS_PER_CF); + } } + } catch (Exception e) { + if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, null, null, null, null, e); + } + callback.onFailure(e); } } private void process(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, Collection cfIds, List cfIdList, MultipleTbCallback callback) { - if (cfIds.contains(ctx.getCfId())) { - callback.onSuccess(CALLBACKS_PER_CF); - } else { - if (proto.getTsDataCount() > 0) { - processTelemetry(ctx, proto, cfIdList, callback); - } else if (proto.getAttrDataCount() > 0) { - processAttributes(ctx, proto, cfIdList, callback); - } else { + try { + if (cfIds.contains(ctx.getCfId())) { callback.onSuccess(CALLBACKS_PER_CF); + } else { + if (proto.getTsDataCount() > 0) { + processTelemetry(ctx, proto, cfIdList, callback); + } else if (proto.getAttrDataCount() > 0) { + processAttributes(ctx, proto, cfIdList, callback); + } else { + callback.onSuccess(CALLBACKS_PER_CF); + } + } + } catch (Exception e) { + if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, null, null, null, null, e); } + callback.onFailure(e); } } @@ -197,18 +218,12 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM state = getOrInitState(ctx); justRestored = true; } - try { - if (state.updateState(newArgValues) || justRestored) { - cfIdList = new ArrayList<>(cfIdList); - cfIdList.add(ctx.getCfId()); - processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback); - } else { - callback.onSuccess(CALLBACKS_PER_CF); - } - } catch (Exception e) { - if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { - systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, null, e); - } + if (state.updateState(newArgValues) || justRestored) { + cfIdList = new ArrayList<>(cfIdList); + cfIdList.add(ctx.getCfId()); + processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback); + } else { + callback.onSuccess(CALLBACKS_PER_CF); } } @@ -218,44 +233,31 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (state != null) { return state; } else { - try { - ListenableFuture stateFuture = systemContext.getCalculatedFieldProcessingService().fetchStateFromDb(ctx, entityId); - // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. - // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. - // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, - // but this will significantly complicate the code. - state = stateFuture.get(1, TimeUnit.MINUTES); - states.put(ctx.getCfId(), state); - } catch (Exception e) { - if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { - systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, null, null, null, null, e); - } - throw new RuntimeException(e); - } + ListenableFuture stateFuture = systemContext.getCalculatedFieldProcessingService().fetchStateFromDb(ctx, entityId); + // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. + // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. + // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, + // but this will significantly complicate the code. + state = stateFuture.get(1, TimeUnit.MINUTES); + states.put(ctx.getCfId(), state); } return state; } @SneakyThrows private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) { - try { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); - if (state.isReady() && ctx.isInitialized()) { - CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); - state.checkStateSize(ctxId, ctx.getMaxStateSizeInKBytes()); - cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); - if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { - systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null); - } - } else { - callback.onSuccess(); // State was updated but no calculation performed; - } - cfStateService.persistState(ctxId, state, callback); - } catch (Exception e) { - if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { - systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, null, e); + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); + if (state.isReady() && ctx.isInitialized()) { + CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); + state.checkStateSize(ctxId, ctx.getMaxStateSizeInKBytes()); + cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); + if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null); } + } else { + callback.onSuccess(); // State was updated but no calculation performed; } + cfStateService.persistState(ctxId, state, callback); } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 92722fb96b..e489c8ecef 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -119,7 +119,8 @@ public class TsRollingArgumentEntry implements ArgumentEntry { } cleanupExpiredRecords(); } catch (Exception e) { - throw new IllegalArgumentException("Time series rolling arguments supports only numeric values."); + log.warn("Time series rolling arguments supports only numeric values."); +// throw new IllegalArgumentException("Time series rolling arguments supports only numeric values."); } } From a0331655d17acf6a72e109751e006e1c2bdc52b7 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 18 Feb 2025 11:08:09 +0200 Subject: [PATCH 11/11] fixed test script --- .../server/actors/ActorSystemContext.java | 32 ++++++----------- .../controller/CalculatedFieldController.java | 15 +++++--- .../controller/SystemInfoController.java | 24 +++++-------- .../cf/ctx/state/TsRollingArgumentEntry.java | 3 +- .../utils/DebugModeRateLimitsConfig.java | 36 +++++++++++++++++++ .../api/tbel/DefaultTbelInvokeService.java | 1 + .../script/api/tbel/TbTimeWindow.java | 36 +++++++++++++++++++ .../script/api/tbel/TbelCfArg.java | 14 ++++++++ .../script/api/tbel/TbelCfSingleValueArg.java | 16 +++++++++ .../script/api/tbel/TbelCfTsRollingArg.java | 26 ++++++++++---- 10 files changed, 154 insertions(+), 49 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/utils/DebugModeRateLimitsConfig.java create mode 100644 common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 14c8a41fb5..1a5e619d2c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -132,6 +132,7 @@ import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import org.thingsboard.server.service.transport.TbCoreToTransportService; +import org.thingsboard.server.utils.DebugModeRateLimitsConfig; import java.io.PrintWriter; import java.io.StringWriter; @@ -453,10 +454,15 @@ public class ActorSystemContext { private ApiLimitService apiLimitService; @Lazy - @Autowired() + @Autowired(required = false) @Getter private RateLimitService rateLimitService; + @Lazy + @Autowired(required = false) + @Getter + private DebugModeRateLimitsConfig debugModeRateLimitsConfig; + /** * The following Service will be null if we operate in tb-core mode */ @@ -614,22 +620,6 @@ public class ActorSystemContext { @Getter private long sessionReportTimeout; - @Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.enabled:true}") - @Getter - private boolean debugPerTenantEnabled; - - @Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") - @Getter - private String debugPerTenantLimitsConfiguration; - - @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.enabled:true}") - @Getter - private boolean calculatedFieldsDebugPerTenantEnabled; - - @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") - @Getter - private String calculatedFieldsDebugPerTenantLimitsConfiguration; - @Value("${actors.rpc.submit_strategy:BURST}") @Getter private String rpcSubmitStrategy; @@ -783,9 +773,9 @@ public class ActorSystemContext { } private boolean checkLimits(TenantId tenantId, TbMsg tbMsg, Throwable error) { - if (debugPerTenantEnabled) { + if (debugModeRateLimitsConfig.isRuleChainDebugPerTenantLimitsEnabled()) { DebugTbRateLimits debugTbRateLimits = debugPerTenantLimits.computeIfAbsent(tenantId, id -> - new DebugTbRateLimits(new TbRateLimits(debugPerTenantLimitsConfiguration), false)); + new DebugTbRateLimits(new TbRateLimits(debugModeRateLimitsConfig.getRuleChainDebugPerTenantLimitsConfiguration()), false)); if (!debugTbRateLimits.getTbRateLimits().tryConsume()) { if (!debugTbRateLimits.isRuleChainEventSaved()) { @@ -852,8 +842,8 @@ public class ActorSystemContext { } private boolean checkLimits(TenantId tenantId) { - if (calculatedFieldsDebugPerTenantEnabled && - !rateLimitService.checkRateLimit(LimitedApi.CALCULATED_FIELD_DEBUG_EVENTS, (Object) tenantId, calculatedFieldsDebugPerTenantLimitsConfiguration)) { + if (debugModeRateLimitsConfig.isCalculatedFieldDebugPerTenantLimitsEnabled() && + !rateLimitService.checkRateLimit(LimitedApi.CALCULATED_FIELD_DEBUG_EVENTS, (Object) tenantId, debugModeRateLimitsConfig.getCalculatedFieldDebugPerTenantLimitsConfiguration())) { log.trace("[{}] Calculated field debug event limits exceeded!", tenantId); return false; } diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 33998801ff..39f7fc91a8 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -33,6 +33,7 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.tbel.TbelCfArg; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; @@ -84,9 +85,15 @@ public class CalculatedFieldController extends BaseController { private static final String TEST_SCRIPT_EXPRESSION = "Execute the Script expression and return the result. The format of request: \n\n" + MARKDOWN_CODE_BLOCK_START + "{\n" + - " \"expression\": \"var temp = 0; foreach(element: temperature.values) {temp += element.value;} var avgTemperature = temp / temperature.values.size(); var adjustedTemperature = avgTemperature + 0.1 * humidity.value; return {\"adjustedTemperature\": adjustedTemperature};\",\n" + + " \"expression\": \"var temp = 0; foreach(element: temperature.values) {temp += element.value;} var avgTemperature = temp / temperature.values.size(); var adjustedTemperature = avgTemperature + 0.1 * humidity.value; return {\\\"adjustedTemperature\\\": adjustedTemperature};\",\n" + " \"arguments\": {\n" + " \"temperature\": {\n" + + " \"type\": \"TS_ROLLING\",\n" + + " \"timeWindow\": {\n" + + " \"startTs\": 1739775630002,\n" + + " \"endTs\": 65432211,\n" + + " \"limit\": 5\n" + + " },\n" + " \"values\": [\n" + " { \"ts\": 1739775639851, \"value\": 23 },\n" + " { \"ts\": 1739775664561, \"value\": 43 },\n" + @@ -95,7 +102,7 @@ public class CalculatedFieldController extends BaseController { " { \"ts\": 1739776228452, \"value\": 22 }\n" + " ]\n" + " },\n" + - " \"humidity\": { \"ts\": 1739776478057, \"value\": 23 }\n" + + " \"humidity\": { \"type\": \"SINGLE_VALUE\", \"ts\": 1739776478057, \"value\": 23 }\n" + " }\n" + "}" + MARKDOWN_CODE_BLOCK_END @@ -176,8 +183,8 @@ public class CalculatedFieldController extends BaseController { @io.swagger.v3.oas.annotations.parameters.RequestBody(description = "Test calculated field TBEL expression.") @RequestBody JsonNode inputParams) { String expression = inputParams.get("expression").asText(); - Map arguments = Objects.requireNonNullElse( - JacksonUtil.convertValue(inputParams.get("arguments"), new TypeReference>() { + Map arguments = Objects.requireNonNullElse( + JacksonUtil.convertValue(inputParams.get("arguments"), new TypeReference>() { }), Collections.emptyMap() ); diff --git a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java index d630c5a22a..3fbedcebaf 100644 --- a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java +++ b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java @@ -46,6 +46,7 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; +import org.thingsboard.server.utils.DebugModeRateLimitsConfig; import java.util.Collections; import java.util.List; @@ -74,18 +75,6 @@ public class SystemInfoController extends BaseController { @Value("${debug.settings.default_duration:15}") private int defaultDebugDurationMinutes; - @Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.enabled:true}") - private boolean ruleChainDebugPerTenantLimitsEnabled; - - @Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") - private String ruleChainDebugPerTenantLimitsConfiguration; - - @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.enabled:true}") - private boolean calculatedFieldDebugPerTenantLimitsEnabled; - - @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") - private String calculatedFieldDebugPerTenantLimitsConfiguration; - @Autowired(required = false) private BuildProperties buildProperties; @@ -95,6 +84,9 @@ public class SystemInfoController extends BaseController { @Autowired private QrCodeSettingService qrCodeSettingService; + @Autowired + private DebugModeRateLimitsConfig debugModeRateLimitsConfig; + @PostConstruct public void init() { JsonNode info = buildInfoObject(); @@ -158,11 +150,11 @@ public class SystemInfoController extends BaseController { DefaultTenantProfileConfiguration tenantProfileConfiguration = tenantProfileCache.get(tenantId).getDefaultProfileConfiguration(); systemParams.setMaxResourceSize(tenantProfileConfiguration.getMaxResourceSize()); systemParams.setMaxDebugModeDurationMinutes(DebugModeUtil.getMaxDebugAllDuration(tenantProfileConfiguration.getMaxDebugModeDurationMinutes(), defaultDebugDurationMinutes)); - if (ruleChainDebugPerTenantLimitsEnabled) { - systemParams.setRuleChainDebugPerTenantLimitsConfiguration(ruleChainDebugPerTenantLimitsConfiguration); + if (debugModeRateLimitsConfig.isRuleChainDebugPerTenantLimitsEnabled()) { + systemParams.setRuleChainDebugPerTenantLimitsConfiguration(debugModeRateLimitsConfig.getRuleChainDebugPerTenantLimitsConfiguration()); } - if (calculatedFieldDebugPerTenantLimitsEnabled) { - systemParams.setCalculatedFieldDebugPerTenantLimitsConfiguration(calculatedFieldDebugPerTenantLimitsConfiguration); + if (debugModeRateLimitsConfig.isCalculatedFieldDebugPerTenantLimitsEnabled()) { + systemParams.setCalculatedFieldDebugPerTenantLimitsConfiguration(debugModeRateLimitsConfig.getCalculatedFieldDebugPerTenantLimitsConfiguration()); } } systemParams.setMobileQrEnabled(Optional.ofNullable(qrCodeSettingService.findQrCodeSettings(TenantId.SYS_TENANT_ID)) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index e489c8ecef..866e2f5e09 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -82,8 +82,7 @@ public class TsRollingArgumentEntry implements ArgumentEntry { for (var e : tsRecords.entrySet()) { values.add(new TbelCfTsDoubleVal(e.getKey(), e.getValue())); } - long ts = System.currentTimeMillis(); - return new TbelCfTsRollingArg(ts - timeWindow, ts, values); + return new TbelCfTsRollingArg(limit, timeWindow, values); } @Override diff --git a/application/src/main/java/org/thingsboard/server/utils/DebugModeRateLimitsConfig.java b/application/src/main/java/org/thingsboard/server/utils/DebugModeRateLimitsConfig.java new file mode 100644 index 0000000000..edae22d1e3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/DebugModeRateLimitsConfig.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +@Data +public class DebugModeRateLimitsConfig { + + @Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.enabled:true}") + private boolean ruleChainDebugPerTenantLimitsEnabled; + @Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") + private String ruleChainDebugPerTenantLimitsConfiguration; + + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.enabled:true}") + private boolean calculatedFieldDebugPerTenantLimitsEnabled; + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") + private String calculatedFieldDebugPerTenantLimitsConfiguration; + +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java index 177bc517a8..2d19e39c31 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java @@ -136,6 +136,7 @@ public class DefaultTbelInvokeService extends AbstractScriptInvokeService implem parserConfig.registerDataType("TbelCfSingleValueArg", TbelCfSingleValueArg.class, TbelCfSingleValueArg::memorySize); parserConfig.registerDataType("TbelCfTsRollingArg", TbelCfTsRollingArg.class, TbelCfTsRollingArg::memorySize); parserConfig.registerDataType("TbelCfTsDoubleVal", TbelCfTsDoubleVal.class, TbelCfTsDoubleVal::memorySize); + parserConfig.registerDataType("TbTimeWindow", TbTimeWindow.class, TbTimeWindow::memorySize); TbUtils.register(parserConfig); executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, "tbel-executor")); try { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java new file mode 100644 index 0000000000..f90a60ff3a --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class TbTimeWindow implements TbelCfObject { + + public static final long OBJ_SIZE = 32L; + + private long startTs; + private long endTs; + private int limit; + + @Override + public long memorySize() { + return OBJ_SIZE; + } + +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java index ac5bd5cc06..7fe75acd05 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java @@ -15,6 +15,20 @@ */ package org.thingsboard.script.api.tbel; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = TbelCfSingleValueArg.class, name = "SINGLE_VALUE"), + @JsonSubTypes.Type(value = TbelCfTsRollingArg.class, name = "TS_ROLLING") +}) public interface TbelCfArg extends TbelCfObject { + String getType(); + } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfSingleValueArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfSingleValueArg.java index 7076e9dd5e..d35ac36ab0 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfSingleValueArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfSingleValueArg.java @@ -15,6 +15,8 @@ */ package org.thingsboard.script.api.tbel; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; @Data @@ -23,9 +25,23 @@ public class TbelCfSingleValueArg implements TbelCfArg { private final long ts; private final Object value; + @JsonCreator + public TbelCfSingleValueArg( + @JsonProperty("ts") long ts, + @JsonProperty("value") Object value + ) { + this.ts = ts; + this.value = value; + } + @Override public long memorySize() { return 8L; // TODO; } + @Override + public String getType() { + return "SINGLE_VALUE"; + } + } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java index 230baaee0a..843abf8d5c 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java @@ -15,7 +15,9 @@ */ package org.thingsboard.script.api.tbel; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Getter; import java.util.Collections; @@ -28,15 +30,22 @@ import static org.thingsboard.script.api.tbel.TbelCfTsDoubleVal.OBJ_SIZE; public class TbelCfTsRollingArg implements TbelCfArg, Iterable { @Getter - private final long startTs; - @Getter - private final long endTs; + private final TbTimeWindow timeWindow; @Getter private final List values; - public TbelCfTsRollingArg(long startTs, long endTs, List values) { - this.startTs = startTs; - this.endTs = endTs; + @JsonCreator + public TbelCfTsRollingArg( + @JsonProperty("timeWindow") TbTimeWindow timeWindow, + @JsonProperty("values") List values + ) { + this.timeWindow = timeWindow; + this.values = Collections.unmodifiableList(values); + } + + public TbelCfTsRollingArg(int limit, long timeWindow, List values) { + long ts = System.currentTimeMillis(); + this.timeWindow = new TbTimeWindow(ts - timeWindow, ts, limit); this.values = Collections.unmodifiableList(values); } @@ -76,4 +85,9 @@ public class TbelCfTsRollingArg implements TbelCfArg, Iterable