Browse Source

Merge improvements

pull/12713/head
Andrii Shvaika 2 years ago
parent
commit
3cdc933b67
  1. 114
      application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
  2. 80
      application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java
  3. 17
      application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java
  4. 25
      application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java
  5. 19
      application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java
  6. 24
      application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java
  7. 141
      application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java
  8. 10
      application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java
  9. 4
      application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java
  10. 39
      application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java
  11. 9
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java
  12. 17
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java
  13. 6
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java
  14. 8
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java
  15. 15
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java
  16. 18
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java
  17. 6
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java
  18. 20
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java
  19. 3
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java
  20. 24
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java
  21. 70
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java
  22. 144
      application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java
  23. 36
      application/src/main/java/org/thingsboard/server/utils/DebugModeRateLimitsConfig.java
  24. 4
      application/src/main/resources/thingsboard.yml
  25. 284
      application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java
  26. 56
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java
  27. 7
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java
  28. 2
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java
  29. 60
      application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java
  30. 1
      common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java
  31. 1
      common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java
  32. 14
      common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java
  33. 3
      common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java
  34. 1
      common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java
  35. 11
      common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java
  36. 16
      common/proto/src/main/proto/queue.proto
  37. 6
      common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java
  38. 12
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java
  39. 12
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java
  40. 6
      common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java
  41. 1
      common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java
  42. 36
      common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java
  43. 14
      common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java
  44. 16
      common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfSingleValueArg.java
  45. 22
      common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java
  46. 4
      dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java
  47. 14
      dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java
  48. 10
      dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java

114
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java

@ -41,6 +41,7 @@ import org.thingsboard.script.api.js.JsInvokeService;
import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.script.api.tbel.TbelInvokeService;
import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.service.ActorService;
import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.actors.tenant.DebugTbRateLimits;
import org.thingsboard.server.cache.limits.RateLimitService;
import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent;
import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.ErrorEvent;
@ -50,6 +51,7 @@ import org.thingsboard.server.common.data.event.RuleNodeDebugEvent;
import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.limit.LimitedApi;
import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbActorMsg;
@ -107,8 +109,8 @@ import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService;
import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache;
import org.thingsboard.server.service.cf.CalculatedFieldStateService; import org.thingsboard.server.service.cf.CalculatedFieldStateService;
import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache;
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry;
import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.component.ComponentDiscoveryService;
import org.thingsboard.server.service.edge.rpc.EdgeRpcService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService;
@ -130,6 +132,7 @@ import org.thingsboard.server.service.state.DeviceStateService;
import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService;
import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService;
import org.thingsboard.server.service.transport.TbCoreToTransportService; import org.thingsboard.server.service.transport.TbCoreToTransportService;
import org.thingsboard.server.utils.DebugModeRateLimitsConfig;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.io.StringWriter; import java.io.StringWriter;
@ -182,8 +185,6 @@ public class ActorSystemContext {
private final ConcurrentMap<TenantId, DebugTbRateLimits> debugPerTenantLimits = new ConcurrentHashMap<>(); private final ConcurrentMap<TenantId, DebugTbRateLimits> debugPerTenantLimits = new ConcurrentHashMap<>();
private final ConcurrentMap<TenantId, TbRateLimits> cfDebugPerTenantLimits = new ConcurrentHashMap<>();
public ConcurrentMap<TenantId, DebugTbRateLimits> getDebugPerTenantLimits() { public ConcurrentMap<TenantId, DebugTbRateLimits> getDebugPerTenantLimits() {
return debugPerTenantLimits; return debugPerTenantLimits;
} }
@ -452,6 +453,16 @@ public class ActorSystemContext {
@Getter @Getter
private ApiLimitService apiLimitService; private ApiLimitService apiLimitService;
@Lazy
@Autowired(required = false)
@Getter
private RateLimitService rateLimitService;
@Lazy
@Autowired(required = false)
@Getter
private DebugModeRateLimitsConfig debugModeRateLimitsConfig;
/** /**
* The following Service will be null if we operate in tb-core mode * The following Service will be null if we operate in tb-core mode
*/ */
@ -609,14 +620,6 @@ public class ActorSystemContext {
@Getter @Getter
private long sessionReportTimeout; private long sessionReportTimeout;
@Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.enabled:true}")
@Getter
private boolean debugPerTenantEnabled;
@Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.configuration:50000:3600}")
@Getter
private String debugPerTenantLimitsConfiguration;
@Value("${actors.rpc.submit_strategy:BURST}") @Value("${actors.rpc.submit_strategy:BURST}")
@Getter @Getter
private String rpcSubmitStrategy; private String rpcSubmitStrategy;
@ -641,14 +644,6 @@ public class ActorSystemContext {
@Getter @Getter
private String deviceStateNodeRateLimitConfig; private String deviceStateNodeRateLimitConfig;
@Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.enabled:true}")
@Getter
private boolean cfDebugPerTenantEnabled;
@Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.configuration:50000:3600}")
@Getter
private String cfDebugPerTenantLimitsConfiguration;
@Getter @Getter
@Setter @Setter
private TbActorSystem actorSystem; private TbActorSystem actorSystem;
@ -778,9 +773,9 @@ public class ActorSystemContext {
} }
private boolean checkLimits(TenantId tenantId, TbMsg tbMsg, Throwable error) { private boolean checkLimits(TenantId tenantId, TbMsg tbMsg, Throwable error) {
if (debugPerTenantEnabled) { if (debugModeRateLimitsConfig.isRuleChainDebugPerTenantLimitsEnabled()) {
DebugTbRateLimits debugTbRateLimits = debugPerTenantLimits.computeIfAbsent(tenantId, id -> DebugTbRateLimits debugTbRateLimits = debugPerTenantLimits.computeIfAbsent(tenantId, id ->
new DebugTbRateLimits(new TbRateLimits(debugPerTenantLimitsConfiguration), false)); new DebugTbRateLimits(new TbRateLimits(debugModeRateLimitsConfig.getRuleChainDebugPerTenantLimitsConfiguration()), false));
if (!debugTbRateLimits.getTbRateLimits().tryConsume()) { if (!debugTbRateLimits.getTbRateLimits().tryConsume()) {
if (!debugTbRateLimits.isRuleChainEventSaved()) { if (!debugTbRateLimits.isRuleChainEventSaved()) {
@ -810,50 +805,51 @@ public class ActorSystemContext {
Futures.addCallback(future, RULE_CHAIN_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); Futures.addCallback(future, RULE_CHAIN_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor());
} }
public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map<String, ArgumentEntry> arguments, public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map<String, ArgumentEntry> arguments, UUID tbMsgId, TbMsgType tbMsgType, String result, Throwable error) {
UUID tbMsgId, TbMsgType tbMsgType, String result, Throwable error) { if (checkLimits(tenantId)) {
if (cfDebugPerTenantEnabled) { try {
TbRateLimits rateLimits = cfDebugPerTenantLimits.computeIfAbsent(tenantId, id -> new TbRateLimits(cfDebugPerTenantLimitsConfiguration)); CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder()
.tenantId(tenantId)
if (rateLimits.tryConsume()) { .entityId(calculatedFieldId.getId())
try { .serviceId(getServiceId())
CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() .calculatedFieldId(calculatedFieldId)
.tenantId(tenantId) .eventEntity(entityId);
.entityId(calculatedFieldId.getId()) if (tbMsgId != null) {
.serviceId(getServiceId()) eventBuilder.msgId(tbMsgId);
.calculatedFieldId(calculatedFieldId)
.eventEntity(entityId);
if (tbMsgId != null) {
eventBuilder.msgId(tbMsgId);
}
if (tbMsgType != null) {
eventBuilder.msgType(tbMsgType.name());
}
if (arguments != null) {
eventBuilder.arguments(JacksonUtil.toString(
arguments.entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toTbelCfArg()))
));
}
if (result != null) {
eventBuilder.result(result);
}
if (error != null) {
eventBuilder.error(toString(error));
}
ListenableFuture<Void> future = eventService.saveAsync(eventBuilder.build());
Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor());
} catch (IllegalArgumentException ex) {
log.warn("Failed to persist calculated field debug message", ex);
} }
if (log.isTraceEnabled()) { if (tbMsgType != null) {
log.trace("[{}] Tenant level debug mode rate limit detected: {}", tenantId, calculatedFieldId); eventBuilder.msgType(tbMsgType.name());
}
if (arguments != null) {
eventBuilder.arguments(JacksonUtil.toString(
arguments.entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toTbelCfArg()))
));
}
if (result != null) {
eventBuilder.result(result);
} }
if (error != null) {
eventBuilder.error(error.getMessage());
}
ListenableFuture<Void> future = eventService.saveAsync(eventBuilder.build());
Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor());
} catch (IllegalArgumentException ex) {
log.warn("Failed to persist calculated field debug message", ex);
} }
} }
} }
private boolean checkLimits(TenantId tenantId) {
if (debugModeRateLimitsConfig.isCalculatedFieldDebugPerTenantLimitsEnabled() &&
!rateLimitService.checkRateLimit(LimitedApi.CALCULATED_FIELD_DEBUG_EVENTS, (Object) tenantId, debugModeRateLimitsConfig.getCalculatedFieldDebugPerTenantLimitsConfiguration())) {
log.trace("[{}] Calculated field debug event limits exceeded!", tenantId);
return false;
}
return true;
}
public static Exception toException(Throwable error) { public static Exception toException(Throwable error) {
return Exception.class.isInstance(error) ? (Exception) error : new Exception(error); return Exception.class.isInstance(error) ? (Exception) error : new Exception(error);
} }

80
application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java

@ -38,8 +38,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelem
import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto;
import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService;
import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.CalculatedFieldResult;
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId;
import org.thingsboard.server.service.cf.CalculatedFieldStateService; import org.thingsboard.server.service.cf.CalculatedFieldStateService;
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId;
import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState;
@ -112,8 +112,16 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
log.info("Force reinitialization of CF: [{}].", cfCtx.getCfId()); log.info("Force reinitialization of CF: [{}].", cfCtx.getCfId());
states.remove(cfCtx.getCfId()); states.remove(cfCtx.getCfId());
} }
var cfState = getOrInitState(cfCtx); try {
processStateIfReady(cfCtx, Collections.singletonList(cfCtx.getCfId()), cfState, null, null, msg.getCallback()); var cfState = getOrInitState(cfCtx);
processStateIfReady(cfCtx, Collections.singletonList(cfCtx.getCfId()), cfState, null, null, msg.getCallback());
} catch (Exception e) {
if (DebugModeUtil.isDebugFailuresAvailable(cfCtx.getCalculatedField())) {
systemContext.persistCalculatedFieldDebugEvent(tenantId, cfCtx.getCfId(), entityId, null, null, null, null, e);
}
msg.getCallback().onFailure(e);
}
} }
public void process(CalculatedFieldEntityDeleteMsg msg) { public void process(CalculatedFieldEntityDeleteMsg msg) {
@ -151,31 +159,45 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
var proto = msg.getProto(); var proto = msg.getProto();
var ctx = msg.getCtx(); var ctx = msg.getCtx();
var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback());
List<CalculatedFieldId> cfIds = getCalculatedFieldIds(proto); try {
if (cfIds.contains(ctx.getCfId())) { List<CalculatedFieldId> cfIds = getCalculatedFieldIds(proto);
callback.onSuccess(CALLBACKS_PER_CF); if (cfIds.contains(ctx.getCfId())) {
} else {
if (proto.getTsDataCount() > 0) {
processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto));
} else if (proto.getAttrDataCount() > 0) {
processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto));
} else {
callback.onSuccess(CALLBACKS_PER_CF); callback.onSuccess(CALLBACKS_PER_CF);
} else {
if (proto.getTsDataCount() > 0) {
processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto));
} else if (proto.getAttrDataCount() > 0) {
processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto));
} else {
callback.onSuccess(CALLBACKS_PER_CF);
}
} }
} catch (Exception e) {
if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) {
systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, null, null, null, null, e);
}
callback.onFailure(e);
} }
} }
private void process(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, Collection<CalculatedFieldId> cfIds, List<CalculatedFieldId> cfIdList, MultipleTbCallback callback) { private void process(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, Collection<CalculatedFieldId> cfIds, List<CalculatedFieldId> cfIdList, MultipleTbCallback callback) {
if (cfIds.contains(ctx.getCfId())) { try {
callback.onSuccess(CALLBACKS_PER_CF); if (cfIds.contains(ctx.getCfId())) {
} else {
if (proto.getTsDataCount() > 0) {
processTelemetry(ctx, proto, cfIdList, callback);
} else if (proto.getAttrDataCount() > 0) {
processAttributes(ctx, proto, cfIdList, callback);
} else {
callback.onSuccess(CALLBACKS_PER_CF); callback.onSuccess(CALLBACKS_PER_CF);
} else {
if (proto.getTsDataCount() > 0) {
processTelemetry(ctx, proto, cfIdList, callback);
} else if (proto.getAttrDataCount() > 0) {
processAttributes(ctx, proto, cfIdList, callback);
} else {
callback.onSuccess(CALLBACKS_PER_CF);
}
}
} catch (Exception e) {
if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) {
systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, null, null, null, null, e);
} }
callback.onFailure(e);
} }
} }
@ -230,22 +252,18 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM
@SneakyThrows @SneakyThrows
private void processStateIfReady(CalculatedFieldCtx ctx, List<CalculatedFieldId> cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) { private void processStateIfReady(CalculatedFieldCtx ctx, List<CalculatedFieldId> cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) {
CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId);
if (state.isReady() && ctx.isInitialized()) { if (state.isReady() && ctx.isInitialized()) {
try { CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS);
CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); state.checkStateSize(ctxId, ctx.getMaxStateSizeInKBytes());
cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback);
if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) {
systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResultMap()), null); systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null);
}
} catch (Exception e) {
if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) {
systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, null, e);
}
} }
} else { } else {
callback.onSuccess(); // State was updated but no calculation performed; callback.onSuccess(); // State was updated but no calculation performed;
} }
cfStateService.persistState(ctx, new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); cfStateService.persistState(ctxId, state, callback);
} }
private Map<String, ArgumentEntry> mapToArguments(CalculatedFieldCtx ctx, List<TsKvProto> data) { private Map<String, ArgumentEntry> mapToArguments(CalculatedFieldCtx ctx, List<TsKvProto> data) {

17
application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java

@ -41,9 +41,9 @@ import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.cf.CalculatedFieldService;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto;
import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService;
import org.thingsboard.server.service.cf.CalculatedFieldStateService;
import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache;
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId;
import org.thingsboard.server.service.cf.CalculatedFieldStateService;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbAssetProfileCache;
import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache;
@ -53,7 +53,6 @@ import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
@ -263,6 +262,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware
callback.onSuccess(); callback.onSuccess();
} else { } else {
var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService());
try {
newCfCtx.init();
} catch (Exception e) {
if (DebugModeUtil.isDebugAllAvailable(newCf)) {
systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e);
}
}
calculatedFields.put(newCf.getId(), newCfCtx); calculatedFields.put(newCf.getId(), newCfCtx);
List<CalculatedFieldCtx> oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); List<CalculatedFieldCtx> oldCfList = entityIdCalculatedFields.get(newCf.getEntityId());
List<CalculatedFieldCtx> newCfList = new ArrayList<>(oldCfList.size()); List<CalculatedFieldCtx> newCfList = new ArrayList<>(oldCfList.size());
@ -287,13 +293,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware
// Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead)
var stateChanges = newCfCtx.hasStateChanges(oldCfCtx); var stateChanges = newCfCtx.hasStateChanges(oldCfCtx);
if (stateChanges || newCfCtx.hasOtherSignificantChanges(oldCfCtx)) { if (stateChanges || newCfCtx.hasOtherSignificantChanges(oldCfCtx)) {
try {
newCfCtx.init();
} catch (Exception e) {
if (DebugModeUtil.isDebugAllAvailable(newCf)) {
systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e);
}
}
initCf(newCfCtx, callback, stateChanges); initCf(newCfCtx, callback, stateChanges);
} else { } else {
callback.onSuccess(); callback.onSuccess();

25
application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java

@ -33,6 +33,7 @@ import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.script.api.tbel.TbelCfArg;
import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.script.api.tbel.TbelInvokeService;
import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.exception.ThingsboardException;
@ -84,14 +85,24 @@ public class CalculatedFieldController extends BaseController {
private static final String TEST_SCRIPT_EXPRESSION = "Execute the Script expression and return the result. The format of request: \n\n" private static final String TEST_SCRIPT_EXPRESSION = "Execute the Script expression and return the result. The format of request: \n\n"
+ MARKDOWN_CODE_BLOCK_START + MARKDOWN_CODE_BLOCK_START
+ "{\n" + + "{\n" +
" \"expression\": \"var temp = 0; foreach(element: temperature.entrySet()) { temp += element.getValue(); } var avgTemperature = temp / temperature.size(); var adjustedTemperature = avgTemperature + 0.1 * humidity; return { \\\"adjustedTemperature\\\": adjustedTemperature };\",\n" + " \"expression\": \"var temp = 0; foreach(element: temperature.values) {temp += element.value;} var avgTemperature = temp / temperature.values.size(); var adjustedTemperature = avgTemperature + 0.1 * humidity.value; return {\\\"adjustedTemperature\\\": adjustedTemperature};\",\n" +
" \"arguments\": {\n" + " \"arguments\": {\n" +
" \"temperature\": {\n" + " \"temperature\": {\n" +
" \"14327856345\": 22.4,\n" + " \"type\": \"TS_ROLLING\",\n" +
" \"14327857298\": 21.9,\n" + " \"timeWindow\": {\n" +
" \"14327857510\": 22.0\n" + " \"startTs\": 1739775630002,\n" +
" \"endTs\": 65432211,\n" +
" \"limit\": 5\n" +
" },\n" +
" \"values\": [\n" +
" { \"ts\": 1739775639851, \"value\": 23 },\n" +
" { \"ts\": 1739775664561, \"value\": 43 },\n" +
" { \"ts\": 1739775713079, \"value\": 15 },\n" +
" { \"ts\": 1739775999522, \"value\": 34 },\n" +
" { \"ts\": 1739776228452, \"value\": 22 }\n" +
" ]\n" +
" },\n" + " },\n" +
" \"humidity\": 42\n" + " \"humidity\": { \"type\": \"SINGLE_VALUE\", \"ts\": 1739776478057, \"value\": 23 }\n" +
" }\n" + " }\n" +
"}" "}"
+ MARKDOWN_CODE_BLOCK_END + MARKDOWN_CODE_BLOCK_END
@ -172,8 +183,8 @@ public class CalculatedFieldController extends BaseController {
@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "Test calculated field TBEL expression.") @io.swagger.v3.oas.annotations.parameters.RequestBody(description = "Test calculated field TBEL expression.")
@RequestBody JsonNode inputParams) { @RequestBody JsonNode inputParams) {
String expression = inputParams.get("expression").asText(); String expression = inputParams.get("expression").asText();
Map<String, Object> arguments = Objects.requireNonNullElse( Map<String, TbelCfArg> arguments = Objects.requireNonNullElse(
JacksonUtil.convertValue(inputParams.get("arguments"), new TypeReference<Map<String, Object>>() { JacksonUtil.convertValue(inputParams.get("arguments"), new TypeReference<Map<String, TbelCfArg>>() {
}), }),
Collections.emptyMap() Collections.emptyMap()
); );

19
application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java

@ -35,8 +35,8 @@ import org.thingsboard.server.common.data.SystemParams;
import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.mobile.qrCodeSettings.QrCodeSettings;
import org.thingsboard.server.common.data.mobile.qrCodeSettings.QRCodeConfig; import org.thingsboard.server.common.data.mobile.qrCodeSettings.QRCodeConfig;
import org.thingsboard.server.common.data.mobile.qrCodeSettings.QrCodeSettings;
import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.settings.UserSettings; import org.thingsboard.server.common.data.settings.UserSettings;
import org.thingsboard.server.common.data.settings.UserSettingsType; import org.thingsboard.server.common.data.settings.UserSettingsType;
@ -46,6 +46,7 @@ import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.security.model.UserPrincipal;
import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import org.thingsboard.server.utils.DebugModeRateLimitsConfig;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
@ -74,12 +75,6 @@ public class SystemInfoController extends BaseController {
@Value("${debug.settings.default_duration:15}") @Value("${debug.settings.default_duration:15}")
private int defaultDebugDurationMinutes; private int defaultDebugDurationMinutes;
@Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.enabled:true}")
private boolean ruleChainDebugPerTenantLimitsEnabled;
@Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.configuration:50000:3600}")
private String ruleChainDebugPerTenantLimitsConfiguration;
@Autowired(required = false) @Autowired(required = false)
private BuildProperties buildProperties; private BuildProperties buildProperties;
@ -89,6 +84,9 @@ public class SystemInfoController extends BaseController {
@Autowired @Autowired
private QrCodeSettingService qrCodeSettingService; private QrCodeSettingService qrCodeSettingService;
@Autowired
private DebugModeRateLimitsConfig debugModeRateLimitsConfig;
@PostConstruct @PostConstruct
public void init() { public void init() {
JsonNode info = buildInfoObject(); JsonNode info = buildInfoObject();
@ -152,8 +150,11 @@ public class SystemInfoController extends BaseController {
DefaultTenantProfileConfiguration tenantProfileConfiguration = tenantProfileCache.get(tenantId).getDefaultProfileConfiguration(); DefaultTenantProfileConfiguration tenantProfileConfiguration = tenantProfileCache.get(tenantId).getDefaultProfileConfiguration();
systemParams.setMaxResourceSize(tenantProfileConfiguration.getMaxResourceSize()); systemParams.setMaxResourceSize(tenantProfileConfiguration.getMaxResourceSize());
systemParams.setMaxDebugModeDurationMinutes(DebugModeUtil.getMaxDebugAllDuration(tenantProfileConfiguration.getMaxDebugModeDurationMinutes(), defaultDebugDurationMinutes)); systemParams.setMaxDebugModeDurationMinutes(DebugModeUtil.getMaxDebugAllDuration(tenantProfileConfiguration.getMaxDebugModeDurationMinutes(), defaultDebugDurationMinutes));
if (ruleChainDebugPerTenantLimitsEnabled) { if (debugModeRateLimitsConfig.isRuleChainDebugPerTenantLimitsEnabled()) {
systemParams.setRuleChainDebugPerTenantLimitsConfiguration(ruleChainDebugPerTenantLimitsConfiguration); systemParams.setRuleChainDebugPerTenantLimitsConfiguration(debugModeRateLimitsConfig.getRuleChainDebugPerTenantLimitsConfiguration());
}
if (debugModeRateLimitsConfig.isCalculatedFieldDebugPerTenantLimitsEnabled()) {
systemParams.setCalculatedFieldDebugPerTenantLimitsConfiguration(debugModeRateLimitsConfig.getCalculatedFieldDebugPerTenantLimitsConfiguration());
} }
} }
systemParams.setMobileQrEnabled(Optional.ofNullable(qrCodeSettingService.findQrCodeSettings(TenantId.SYS_TENANT_ID)) systemParams.setMobileQrEnabled(Optional.ofNullable(qrCodeSettingService.findQrCodeSettings(TenantId.SYS_TENANT_ID))

24
application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java

@ -0,0 +1,24 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.exception;
public class CalculatedFieldStateException extends RuntimeException {
public CalculatedFieldStateException(String message) {
super(message);
}
}

141
application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java

@ -18,32 +18,14 @@ package org.thingsboard.server.service.cf;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.BasicKvEntry;
import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.util.KvProtoUtil; import org.thingsboard.server.exception.CalculatedFieldStateException;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto;
import org.thingsboard.server.gen.transport.TransportProtos.TsDoubleValProto;
import org.thingsboard.server.gen.transport.TransportProtos.TsRollingArgumentProto;
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState;
import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState;
import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState;
import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry;
import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry;
import java.util.Optional; import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto;
import java.util.TreeMap; import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto;
import java.util.UUID;
public abstract class AbstractCalculatedFieldStateService implements CalculatedFieldStateService { public abstract class AbstractCalculatedFieldStateService implements CalculatedFieldStateService {
@ -51,15 +33,14 @@ public abstract class AbstractCalculatedFieldStateService implements CalculatedF
private ActorSystemContext actorSystemContext; private ActorSystemContext actorSystemContext;
@Override @Override
public final void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { public final void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) {
CalculatedFieldStateMsgProto stateMsg = toProto(stateId, state); if (state.isStateTooLarge()) {
long maxStateSizeInKBytes = ctx.getMaxStateSizeInKBytes(); throw new CalculatedFieldStateException("State size exceeds the maximum allowed limit. The state will not be persisted to RocksDB.");
if (maxStateSizeInKBytes <= 0 || stateMsg.getSerializedSize() <= maxStateSizeInKBytes) {
doPersist(stateId, stateMsg, callback);
} }
doPersist(stateId, toProto(stateId, state), callback);
} }
protected abstract void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateMsgProto stateMsgProto, TbCallback callback); protected abstract void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback);
@Override @Override
public final void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback) { public final void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback) {
@ -68,110 +49,12 @@ public abstract class AbstractCalculatedFieldStateService implements CalculatedF
protected abstract void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback); protected abstract void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback);
protected void processRestoredState(CalculatedFieldStateMsgProto stateMsg) { protected void processRestoredState(CalculatedFieldStateProto stateMsg) {
CalculatedFieldEntityCtxId stateId = fromProto(stateMsg.getId()); var id = fromProto(stateMsg.getId());
CalculatedFieldState state = stateMsg.hasState() ? fromProto(stateMsg.getState()) : null; var state = fromProto(stateMsg);
actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(stateId, state)); actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(id, state));
} }
protected CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) {
return CalculatedFieldEntityCtxIdProto.newBuilder()
.setTenantIdMSB(ctxId.tenantId().getId().getMostSignificantBits())
.setTenantIdLSB(ctxId.tenantId().getId().getLeastSignificantBits())
.setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits())
.setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits())
.setEntityType(ctxId.entityId().getEntityType().name())
.setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits())
.setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits())
.build();
}
protected CalculatedFieldEntityCtxId fromProto(CalculatedFieldEntityCtxIdProto ctxIdProto) {
TenantId tenantId = TenantId.fromUUID(new UUID(ctxIdProto.getTenantIdMSB(), ctxIdProto.getTenantIdLSB()));
EntityId entityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB()));
CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB()));
return new CalculatedFieldEntityCtxId(tenantId, calculatedFieldId, entityId);
}
protected CalculatedFieldStateMsgProto toProto(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state) {
var stateProto = CalculatedFieldStateProto.newBuilder()
.setType(state.getType().name());
state.getArguments().forEach((argName, argEntry) -> {
if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) {
stateProto.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry));
} else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) {
stateProto.addRollingValueArguments(toRollingArgumentProto(argName, rollingArgumentEntry));
}
});
return CalculatedFieldStateMsgProto.newBuilder()
.setId(toProto(stateId))
.setState(stateProto)
.build();
}
private TransportProtos.SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) {
TransportProtos.SingleValueArgumentProto.Builder builder = TransportProtos.SingleValueArgumentProto.newBuilder()
.setArgName(argName);
if (entry != SingleValueArgumentEntry.EMPTY) {
builder.setValue(KvProtoUtil.toTsValueProto(entry.getTs(), entry.getKvEntryValue()));
}
Optional.ofNullable(entry.getVersion()).ifPresent(builder::setVersion);
return builder.build();
}
private TsRollingArgumentProto toRollingArgumentProto(String argName, TsRollingArgumentEntry entry) {
TsRollingArgumentProto.Builder builder = TsRollingArgumentProto.newBuilder().setKey(argName);
if (entry != TsRollingArgumentEntry.EMPTY) {
entry.getTsRecords().forEach((ts, value) -> builder.addTsValue(TsDoubleValProto.newBuilder().setTs(ts).setValue(value).build()));
}
return builder.build();
}
private CalculatedFieldState fromProto(CalculatedFieldStateProto proto) {
if (StringUtils.isEmpty(proto.getType())) {
return null;
}
CalculatedFieldType type = CalculatedFieldType.valueOf(proto.getType());
CalculatedFieldState state = switch (type) {
case SIMPLE -> new SimpleCalculatedFieldState();
case SCRIPT -> new ScriptCalculatedFieldState();
};
proto.getSingleValueArgumentsList().forEach(argProto ->
state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto)));
if (CalculatedFieldType.SCRIPT.equals(type)) {
proto.getRollingValueArgumentsList().forEach(argProto ->
state.getArguments().put(argProto.getKey(), fromRollingArgumentProto(argProto)));
}
return state;
}
private SingleValueArgumentEntry fromSingleValueArgumentProto(TransportProtos.SingleValueArgumentProto proto) {
if (!proto.hasValue()) {
return (SingleValueArgumentEntry) SingleValueArgumentEntry.EMPTY;
}
TransportProtos.TsValueProto tsValueProto = proto.getValue();
long ts = tsValueProto.getTs();
BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto);
return new SingleValueArgumentEntry(ts, kvEntry, proto.getVersion());
}
private TsRollingArgumentEntry fromRollingArgumentProto(TsRollingArgumentProto proto) {
if (proto.getTsValueCount() <= 0) {
return (TsRollingArgumentEntry) TsRollingArgumentEntry.EMPTY;
}
TreeMap<Long, Double> tsRecords = new TreeMap<>();
proto.getTsValueList().forEach(tsValueProto -> tsRecords.put(tsValueProto.getTs(), tsValueProto.getValue()));
return new TsRollingArgumentEntry(tsRecords);
}
} }

10
application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java

@ -15,22 +15,16 @@
*/ */
package org.thingsboard.server.service.cf; package org.thingsboard.server.service.cf;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.Data; import lombok.Data;
import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.OutputType;
import java.util.Map;
@Data @Data
public final class CalculatedFieldResult { public final class CalculatedFieldResult {
private final OutputType type; private final OutputType type;
private final AttributeScope scope; private final AttributeScope scope;
private final Map<String, Object> resultMap; private final JsonNode result;
public CalculatedFieldResult(OutputType type, AttributeScope scope, Map<String, Object> resultMap) {
this.type = type;
this.scope = scope;
this.resultMap = resultMap == null ? Map.of() : Map.copyOf(resultMap);
}
} }

4
application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java

@ -17,15 +17,15 @@ package org.thingsboard.server.service.cf;
import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.exception.CalculatedFieldStateException;
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState;
import java.util.Set; import java.util.Set;
public interface CalculatedFieldStateService { public interface CalculatedFieldStateService {
void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) throws CalculatedFieldStateException;
void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback); void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback);

39
application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java

@ -15,8 +15,6 @@
*/ */
package org.thingsboard.server.service.cf; package org.thingsboard.server.service.cf;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.ListeningExecutorService;
@ -60,8 +58,6 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.dao.usagerecord.ApiLimitService;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto;
@ -92,6 +88,7 @@ import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static org.thingsboard.server.common.data.DataConstants.SCOPE; import static org.thingsboard.server.common.data.DataConstants.SCOPE;
import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto;
@TbRuleEngineComponent @TbRuleEngineComponent
@Service @Service
@ -156,8 +153,7 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP
OutputType type = calculatedFieldResult.getType(); OutputType type = calculatedFieldResult.getType();
TbMsgType msgType = OutputType.ATTRIBUTES.equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; TbMsgType msgType = OutputType.ATTRIBUTES.equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST;
TbMsgMetaData md = OutputType.ATTRIBUTES.equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; TbMsgMetaData md = OutputType.ATTRIBUTES.equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY;
ObjectNode payload = createJsonPayload(calculatedFieldResult); TbMsg msg = TbMsg.newMsg().type(msgType).originator(entityId).previousCalculatedFieldIds(cfIds).metaData(md).data(JacksonUtil.writeValueAsString(calculatedFieldResult.getResult())).build();
TbMsg msg = TbMsg.newMsg().type(msgType).originator(entityId).previousCalculatedFieldIds(cfIds).metaData(md).data(JacksonUtil.writeValueAsString(payload)).build();
clusterService.pushMsgToRuleEngine(tenantId, entityId, msg, new TbQueueCallback() { clusterService.pushMsgToRuleEngine(tenantId, entityId, msg, new TbQueueCallback() {
@Override @Override
public void onSuccess(TbQueueMsgMetadata metadata) { public void onSuccess(TbQueueMsgMetadata metadata) {
@ -229,19 +225,6 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP
return builder.build(); return builder.build();
} }
//TODO: IM: move to utils;
private CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) {
return CalculatedFieldEntityCtxIdProto.newBuilder()
.setTenantIdMSB(ctxId.tenantId().getId().getMostSignificantBits())
.setTenantIdLSB(ctxId.tenantId().getId().getLeastSignificantBits())
.setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits())
.setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits())
.setEntityType(ctxId.entityId().getEntityType().name())
.setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits())
.setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits())
.build();
}
private ListenableFuture<ArgumentEntry> fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { private ListenableFuture<ArgumentEntry> fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) {
return switch (argument.getRefEntityKey().getType()) { return switch (argument.getRefEntityKey().getType()) {
case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument); case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument);
@ -264,7 +247,7 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP
if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { if (kvEntry.isPresent() && kvEntry.get().getValue() != null) {
return ArgumentEntry.createSingleValueArgument(kvEntry.get()); return ArgumentEntry.createSingleValueArgument(kvEntry.get());
} else { } else {
return SingleValueArgumentEntry.EMPTY; return new SingleValueArgumentEntry();
} }
}, calculatedFieldCallbackExecutor); }, calculatedFieldCallbackExecutor);
} }
@ -279,7 +262,7 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP
ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, currentTime, 0, limit, Aggregation.NONE);
ListenableFuture<List<TsKvEntry>> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); ListenableFuture<List<TsKvEntry>> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query));
return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? new TsRollingArgumentEntry(limit, timeWindow) : ArgumentEntry.createTsRollingArgument(tsRolling, limit, timeWindow), calculatedFieldCallbackExecutor);
} }
private KvEntry createDefaultKvEntry(Argument argument) { private KvEntry createDefaultKvEntry(Argument argument) {
@ -297,13 +280,6 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP
return new StringDataEntry(key, defaultValue); return new StringDataEntry(key, defaultValue);
} }
private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) {
ObjectNode payload = JacksonUtil.newObjectNode();
Map<String, Object> resultMap = calculatedFieldResult.getResultMap();
resultMap.forEach((k, v) -> payload.set(k, JacksonUtil.convertValue(v, JsonNode.class)));
return payload;
}
private CalculatedFieldState createStateByType(CalculatedFieldCtx ctx) { private CalculatedFieldState createStateByType(CalculatedFieldCtx ctx) {
return switch (ctx.getCfType()) { return switch (ctx.getCfType()) {
case SIMPLE -> new SimpleCalculatedFieldState(ctx.getArgNames()); case SIMPLE -> new SimpleCalculatedFieldState(ctx.getArgNames());
@ -311,13 +287,6 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP
}; };
} }
private CalculatedFieldIdProto toProto(CalculatedFieldId cfId) {
return CalculatedFieldIdProto.newBuilder()
.setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits())
.setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits())
.build();
}
private static class TbCallbackWrapper implements TbQueueCallback { private static class TbCallbackWrapper implements TbQueueCallback {
private final TbCallback callback; private final TbCallback callback;

9
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java

@ -42,17 +42,16 @@ public interface ArgumentEntry {
boolean updateEntry(ArgumentEntry entry); boolean updateEntry(ArgumentEntry entry);
boolean isEmpty();
static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) {
return new SingleValueArgumentEntry(kvEntry); return new SingleValueArgumentEntry(kvEntry);
} }
static ArgumentEntry createTsRollingArgument(List<TsKvEntry> kvEntries) { static ArgumentEntry createTsRollingArgument(List<TsKvEntry> kvEntries, int limit, long timeWindow) {
return new TsRollingArgumentEntry(kvEntries); return new TsRollingArgumentEntry(kvEntries, limit, timeWindow);
} }
@JsonIgnore
ArgumentEntry copy();
TbelCfArg toTbelCfArg(); TbelCfArg toTbelCfArg();
} }

17
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java

@ -17,6 +17,8 @@ package org.thingsboard.server.service.cf.ctx.state;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId;
import org.thingsboard.server.utils.CalculatedFieldUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
@ -29,6 +31,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState {
protected List<String> requiredArguments; protected List<String> requiredArguments;
protected Map<String, ArgumentEntry> arguments; protected Map<String, ArgumentEntry> arguments;
protected boolean stateTooLarge;
public BaseCalculatedFieldState(List<String> requiredArguments) { public BaseCalculatedFieldState(List<String> requiredArguments) {
this.requiredArguments = requiredArguments; this.requiredArguments = requiredArguments;
@ -36,7 +39,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState {
} }
public BaseCalculatedFieldState() { public BaseCalculatedFieldState() {
this(new ArrayList<>(), new HashMap<>()); this(new ArrayList<>(), new HashMap<>(), false);
} }
@Override @Override
@ -52,7 +55,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState {
ArgumentEntry newEntry = entry.getValue(); ArgumentEntry newEntry = entry.getValue();
ArgumentEntry existingEntry = arguments.get(key); ArgumentEntry existingEntry = arguments.get(key);
if (existingEntry == null || existingEntry == SingleValueArgumentEntry.EMPTY || existingEntry == TsRollingArgumentEntry.EMPTY) { if (existingEntry == null) {
validateNewEntry(newEntry); validateNewEntry(newEntry);
arguments.put(key, newEntry); arguments.put(key, newEntry);
stateUpdated = true; stateUpdated = true;
@ -67,8 +70,14 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState {
@Override @Override
public boolean isReady() { public boolean isReady() {
return arguments.keySet().containsAll(requiredArguments) && return arguments.keySet().containsAll(requiredArguments) &&
!arguments.containsValue(SingleValueArgumentEntry.EMPTY) && arguments.values().stream().noneMatch(ArgumentEntry::isEmpty);
!arguments.containsValue(TsRollingArgumentEntry.EMPTY); }
@Override
public void checkStateSize(CalculatedFieldEntityCtxId ctxId, long maxStateSize) {
if (maxStateSize > 0 && CalculatedFieldUtils.toProto(ctxId, this).getSerializedSize() > maxStateSize) {
setStateTooLarge(true);
}
} }
protected abstract void validateNewEntry(ArgumentEntry newEntry); protected abstract void validateNewEntry(ArgumentEntry newEntry);

6
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java

@ -18,18 +18,12 @@ package org.thingsboard.server.service.cf.ctx.state;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import java.util.Map;
public interface CalculatedFieldScriptEngine { public interface CalculatedFieldScriptEngine {
ListenableFuture<Object> executeScriptAsync(Object[] args); ListenableFuture<Object> executeScriptAsync(Object[] args);
ListenableFuture<Map<String, Object>> executeToMapAsync(Object[] args);
ListenableFuture<JsonNode> executeJsonAsync(Object[] args); ListenableFuture<JsonNode> executeJsonAsync(Object[] args);
ListenableFuture<Map<String, Object>> executeToMapTransform(Object result);
void destroy(); void destroy();
} }

8
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java

@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.CalculatedFieldResult;
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -41,8 +42,6 @@ public interface CalculatedFieldState {
Map<String, ArgumentEntry> getArguments(); Map<String, ArgumentEntry> getArguments();
List<String> getRequiredArguments();
void setRequiredArguments(List<String> requiredArguments); void setRequiredArguments(List<String> requiredArguments);
boolean updateState(Map<String, ArgumentEntry> argumentValues); boolean updateState(Map<String, ArgumentEntry> argumentValues);
@ -51,4 +50,9 @@ public interface CalculatedFieldState {
@JsonIgnore @JsonIgnore
boolean isReady(); boolean isReady();
boolean isStateTooLarge();
void checkStateSize(CalculatedFieldEntityCtxId ctxId, long maxStateSize);
} }

15
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java

@ -19,7 +19,6 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.MoreExecutors;
import io.jsonwebtoken.lang.Collections;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.script.api.ScriptType; import org.thingsboard.script.api.ScriptType;
@ -27,7 +26,6 @@ import org.thingsboard.script.api.tbel.TbelInvokeService;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import javax.script.ScriptException; import javax.script.ScriptException;
import java.util.Map;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
@ -72,24 +70,11 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng
}, MoreExecutors.directExecutor()); }, MoreExecutors.directExecutor());
} }
@Override
public ListenableFuture<Map<String, Object>> executeToMapAsync(Object[] args) {
return Futures.transformAsync(executeScriptAsync(args), this::executeToMapTransform, MoreExecutors.directExecutor());
}
@Override @Override
public ListenableFuture<JsonNode> executeJsonAsync(Object[] args) { public ListenableFuture<JsonNode> executeJsonAsync(Object[] args) {
return Futures.transform(executeScriptAsync(args), JacksonUtil::valueToTree, MoreExecutors.directExecutor()); return Futures.transform(executeScriptAsync(args), JacksonUtil::valueToTree, MoreExecutors.directExecutor());
} }
@Override
public ListenableFuture<Map<String, Object>> executeToMapTransform(Object result) {
if (result instanceof Map) {
return Futures.immediateFuture((Map<String, Object>) result);
}
throw new IllegalArgumentException("Wrong result type: [" + result.getClass().getName() + "]");
}
@Override @Override
public void destroy() { public void destroy() {
tbelInvokeService.release(this.scriptId); tbelInvokeService.release(this.scriptId);

18
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java

@ -26,7 +26,7 @@ import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto;
import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueCallback;
import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.TbQueueMsgMetadata;
import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg;
@ -60,8 +60,8 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta
@Value("${queue.calculated_fields.consumer_per_partition:true}") @Value("${queue.calculated_fields.consumer_per_partition:true}")
private boolean consumerPerPartition; private boolean consumerPerPartition;
private MainQueueConsumerManager<TbProtoQueueMsg<CalculatedFieldStateMsgProto>, CalculatedFieldQueueConfig> stateConsumer; private MainQueueConsumerManager<TbProtoQueueMsg<CalculatedFieldStateProto>, CalculatedFieldQueueConfig> stateConsumer;
private TbKafkaProducerTemplate<TbProtoQueueMsg<CalculatedFieldStateMsgProto>> stateProducer; private TbKafkaProducerTemplate<TbProtoQueueMsg<CalculatedFieldStateProto>> stateProducer;
protected ExecutorService consumersExecutor; protected ExecutorService consumersExecutor;
protected ExecutorService mgmtExecutor; protected ExecutorService mgmtExecutor;
@ -75,11 +75,11 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta
this.mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(Math.max(Runtime.getRuntime().availableProcessors(), 4), "cf-state-mgmt"); this.mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(Math.max(Runtime.getRuntime().availableProcessors(), 4), "cf-state-mgmt");
this.scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("cf-state-consumer-scheduler"); this.scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("cf-state-consumer-scheduler");
this.stateConsumer = MainQueueConsumerManager.<TbProtoQueueMsg<CalculatedFieldStateMsgProto>, CalculatedFieldQueueConfig>builder() this.stateConsumer = MainQueueConsumerManager.<TbProtoQueueMsg<CalculatedFieldStateProto>, CalculatedFieldQueueConfig>builder()
.queueKey(QueueKey.CF_STATES) .queueKey(QueueKey.CF_STATES)
.config(CalculatedFieldQueueConfig.of(consumerPerPartition, (int) pollInterval)) .config(CalculatedFieldQueueConfig.of(consumerPerPartition, (int) pollInterval))
.msgPackProcessor((msgs, consumer, config) -> { .msgPackProcessor((msgs, consumer, config) -> {
for (TbProtoQueueMsg<CalculatedFieldStateMsgProto> msg : msgs) { for (TbProtoQueueMsg<CalculatedFieldStateProto> msg : msgs) {
try { try {
processRestoredState(msg.getValue()); processRestoredState(msg.getValue());
} catch (Throwable t) { } catch (Throwable t) {
@ -97,11 +97,11 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta
.scheduler(scheduler) .scheduler(scheduler)
.taskExecutor(mgmtExecutor) .taskExecutor(mgmtExecutor)
.build(); .build();
this.stateProducer = (TbKafkaProducerTemplate<TbProtoQueueMsg<CalculatedFieldStateMsgProto>>) queueFactory.createCalculatedFieldStateProducer(); this.stateProducer = (TbKafkaProducerTemplate<TbProtoQueueMsg<CalculatedFieldStateProto>>) queueFactory.createCalculatedFieldStateProducer();
} }
@Override @Override
protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateMsgProto stateMsgProto, TbCallback callback) { protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback) {
TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF_STATES, stateId.entityId()); TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF_STATES, stateId.entityId());
stateProducer.send(tpi, stateId.toKey(), new TbProtoQueueMsg<>(stateId.entityId().getId(), stateMsgProto), new TbQueueCallback() { stateProducer.send(tpi, stateId.toKey(), new TbProtoQueueMsg<>(stateId.entityId().getId(), stateMsgProto), new TbQueueCallback() {
@Override @Override
@ -122,9 +122,7 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta
@Override @Override
protected void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback) { protected void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback) {
doPersist(stateId, CalculatedFieldStateMsgProto.newBuilder() //TODO: vklimov
.setId(toProto(stateId))
.build(), callback);
} }
@Override @Override

6
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java

@ -22,7 +22,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto;
import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService; import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService;
import org.thingsboard.server.service.cf.CfRocksDb; import org.thingsboard.server.service.cf.CfRocksDb;
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId;
@ -40,7 +40,7 @@ public class RocksDBCalculatedFieldStateService extends AbstractCalculatedFieldS
private Set<TopicPartitionInfo> partitions; private Set<TopicPartitionInfo> partitions;
@Override @Override
protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateMsgProto stateMsgProto, TbCallback callback) { protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback) {
cfRocksDb.put(stateId.toKey(), stateMsgProto.toByteArray()); cfRocksDb.put(stateId.toKey(), stateMsgProto.toByteArray());
callback.onSuccess(); callback.onSuccess();
} }
@ -61,7 +61,7 @@ public class RocksDBCalculatedFieldStateService extends AbstractCalculatedFieldS
cfRocksDb.forEach((key, value) -> { cfRocksDb.forEach((key, value) -> {
try { try {
processRestoredState(CalculatedFieldStateMsgProto.parseFrom(value)); processRestoredState(CalculatedFieldStateProto.parseFrom(value));
} catch (InvalidProtocolBufferException e) { } catch (InvalidProtocolBufferException e) {
log.error("[{}] Failed to process restored state", key, e); log.error("[{}] Failed to process restored state", key, e);
} }

20
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java

@ -15,6 +15,7 @@
*/ */
package org.thingsboard.server.service.cf.ctx.state; package org.thingsboard.server.service.cf.ctx.state;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.MoreExecutors;
@ -22,18 +23,11 @@ import lombok.Data;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.thingsboard.script.api.tbel.TbelCfArg; import org.thingsboard.script.api.tbel.TbelCfArg;
import org.thingsboard.script.api.tbel.TbelCfSingleValueArg;
import org.thingsboard.script.api.tbel.TbelCfTsDoubleVal;
import org.thingsboard.script.api.tbel.TbelCfTsRollingArg;
import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Argument;
import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.Output;
import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.CalculatedFieldResult;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.TreeMap;
@Data @Data
@Slf4j @Slf4j
@ -55,21 +49,11 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState {
@Override @Override
public ListenableFuture<CalculatedFieldResult> performCalculation(CalculatedFieldCtx ctx) { public ListenableFuture<CalculatedFieldResult> performCalculation(CalculatedFieldCtx ctx) {
arguments.forEach((key, argumentEntry) -> {
if (argumentEntry instanceof TsRollingArgumentEntry tsRollingEntry) {
Argument argument = ctx.getArguments().get(key);
TreeMap<Long, Double> tsRecords = tsRollingEntry.getTsRecords();
if (tsRecords.size() > argument.getLimit()) {
tsRecords.pollFirstEntry();
}
tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow());
}
});
Object[] args = ctx.getArgNames().stream() Object[] args = ctx.getArgNames().stream()
.map(this::toTbelArgument) .map(this::toTbelArgument)
.toArray(); .toArray();
ListenableFuture<Map<String, Object>> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); ListenableFuture<JsonNode> resultFuture = ctx.getCalculatedFieldScriptEngine().executeJsonAsync(args);
Output output = ctx.getOutput(); Output output = ctx.getOutput();
return Futures.transform(resultFuture, return Futures.transform(resultFuture,
result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), result -> new CalculatedFieldResult(output.getType(), output.getScope(), result),

3
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java

@ -19,6 +19,7 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.Output;
import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.data.kv.BasicKvEntry;
@ -63,7 +64,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState {
double expressionResult = expr.evaluate(); double expressionResult = expr.evaluate();
Output output = ctx.getOutput(); Output output = ctx.getOutput();
return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), Map.of(output.getName(), expressionResult))); return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), JacksonUtil.valueToTree(Map.of(output.getName(), expressionResult))));
} }
} }

24
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java

@ -34,8 +34,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto;
@AllArgsConstructor @AllArgsConstructor
public class SingleValueArgumentEntry implements ArgumentEntry { public class SingleValueArgumentEntry implements ArgumentEntry {
public static final ArgumentEntry EMPTY = new SingleValueArgumentEntry(0);
private long ts; private long ts;
private BasicKvEntry kvEntryValue; private BasicKvEntry kvEntryValue;
private Long version; private Long version;
@ -63,27 +61,19 @@ public class SingleValueArgumentEntry implements ArgumentEntry {
this.kvEntryValue = ProtoUtils.basicKvEntryFromKvEntry(entry); this.kvEntryValue = ProtoUtils.basicKvEntryFromKvEntry(entry);
} }
/**
* Internal constructor to create immutable SingleValueArgumentEntry.EMPTY
* */
private SingleValueArgumentEntry(int ignored) {
this.ts = System.currentTimeMillis();
this.kvEntryValue = null;
}
@Override @Override
public ArgumentEntryType getType() { public ArgumentEntryType getType() {
return ArgumentEntryType.SINGLE_VALUE; return ArgumentEntryType.SINGLE_VALUE;
} }
@JsonIgnore @Override
public Object getValue() { public boolean isEmpty() {
return kvEntryValue.getValue(); return kvEntryValue == null;
} }
@Override @JsonIgnore
public ArgumentEntry copy() { public Object getValue() {
return new SingleValueArgumentEntry(this.ts, this.kvEntryValue, this.version); return isEmpty() ? null : kvEntryValue.getValue();
} }
@Override @Override
@ -105,7 +95,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry {
// TODO: should we persist updated ts and version values? // TODO: should we persist updated ts and version values?
BasicKvEntry newValue = singleValueEntry.getKvEntryValue(); BasicKvEntry newValue = singleValueEntry.getKvEntryValue();
if (this.kvEntryValue.getValue().equals(newValue.getValue())) { if (this.kvEntryValue != null && this.kvEntryValue.getValue().equals(newValue.getValue())) {
return false; return false;
} }
this.kvEntryValue = singleValueEntry.getKvEntryValue(); this.kvEntryValue = singleValueEntry.getKvEntryValue();

70
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java

@ -20,21 +20,17 @@ import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.math.NumberUtils;
import org.thingsboard.script.api.tbel.TbelCfArg; import org.thingsboard.script.api.tbel.TbelCfArg;
import org.thingsboard.script.api.tbel.TbelCfTsDoubleVal; import org.thingsboard.script.api.tbel.TbelCfTsDoubleVal;
import org.thingsboard.script.api.tbel.TbelCfTsRollingArg; import org.thingsboard.script.api.tbel.TbelCfTsRollingArg;
import org.thingsboard.server.common.data.kv.BasicKvEntry;
import org.thingsboard.server.common.data.kv.DataType;
import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.exception.CalculatedFieldStateException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.stream.Collectors;
@Data @Data
@NoArgsConstructor @NoArgsConstructor
@ -42,21 +38,26 @@ import java.util.stream.Collectors;
@Slf4j @Slf4j
public class TsRollingArgumentEntry implements ArgumentEntry { public class TsRollingArgumentEntry implements ArgumentEntry {
public static final ArgumentEntry EMPTY = new TsRollingArgumentEntry(0); private Integer limit;
private Long timeWindow;
private static final int MAX_ROLLING_ARGUMENT_ENTRY_SIZE = 1000;
private TreeMap<Long, Double> tsRecords = new TreeMap<>(); private TreeMap<Long, Double> tsRecords = new TreeMap<>();
public TsRollingArgumentEntry(List<TsKvEntry> kvEntries) { public TsRollingArgumentEntry(List<TsKvEntry> kvEntries, int limit, long timeWindow) {
this.limit = limit;
this.timeWindow = timeWindow;
kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry)); kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry));
} }
/** public TsRollingArgumentEntry(TreeMap<Long, Double> tsRecords, int limit, long timeWindow) {
* Internal constructor to create immutable TsRollingArgumentEntry.EMPTY this.tsRecords = tsRecords;
*/ this.limit = limit;
private TsRollingArgumentEntry(int ignored) { this.timeWindow = timeWindow;
}
public TsRollingArgumentEntry(int limit, long timeWindow) {
this.tsRecords = new TreeMap<>(); this.tsRecords = new TreeMap<>();
this.limit = limit;
this.timeWindow = timeWindow;
} }
@Override @Override
@ -64,15 +65,15 @@ public class TsRollingArgumentEntry implements ArgumentEntry {
return ArgumentEntryType.TS_ROLLING; return ArgumentEntryType.TS_ROLLING;
} }
@JsonIgnore
@Override @Override
public Object getValue() { public boolean isEmpty() {
return tsRecords; return tsRecords.isEmpty();
} }
@JsonIgnore
@Override @Override
public ArgumentEntry copy() { public Object getValue() {
return new TsRollingArgumentEntry(new TreeMap<>(tsRecords)); return tsRecords;
} }
@Override @Override
@ -81,11 +82,11 @@ public class TsRollingArgumentEntry implements ArgumentEntry {
for (var e : tsRecords.entrySet()) { for (var e : tsRecords.entrySet()) {
values.add(new TbelCfTsDoubleVal(e.getKey(), e.getValue())); values.add(new TbelCfTsDoubleVal(e.getKey(), e.getValue()));
} }
return new TbelCfTsRollingArg(values); return new TbelCfTsRollingArg(limit, timeWindow, values);
} }
@Override @Override
public boolean updateEntry(ArgumentEntry entry) { public boolean updateEntry(ArgumentEntry entry) throws CalculatedFieldStateException {
if (entry instanceof TsRollingArgumentEntry tsRollingEntry) { if (entry instanceof TsRollingArgumentEntry tsRollingEntry) {
updateTsRollingEntry(tsRollingEntry); updateTsRollingEntry(tsRollingEntry);
} else if (entry instanceof SingleValueArgumentEntry singleValueEntry) { } else if (entry instanceof SingleValueArgumentEntry singleValueEntry) {
@ -107,26 +108,31 @@ public class TsRollingArgumentEntry implements ArgumentEntry {
} }
private void addTsRecord(Long ts, KvEntry value) { private void addTsRecord(Long ts, KvEntry value) {
switch (value.getDataType()) { try {
case LONG -> value.getLongValue().ifPresent(aLong -> tsRecords.put(ts, aLong.doubleValue())); switch (value.getDataType()) {
case DOUBLE -> value.getDoubleValue().ifPresent(aDouble -> tsRecords.put(ts, aDouble)); case LONG -> value.getLongValue().ifPresent(aLong -> tsRecords.put(ts, aLong.doubleValue()));
case BOOLEAN -> value.getBooleanValue().ifPresent(aBoolean -> tsRecords.put(ts, aBoolean ? 1.0 : 0.0)); case DOUBLE -> value.getDoubleValue().ifPresent(aDouble -> tsRecords.put(ts, aDouble));
case STRING -> value.getStrValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); case BOOLEAN -> value.getBooleanValue().ifPresent(aBoolean -> tsRecords.put(ts, aBoolean ? 1.0 : 0.0));
case JSON -> value.getJsonValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); case STRING -> value.getStrValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString)));
//TODO: try catch case JSON -> value.getJsonValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString)));
}
cleanupExpiredRecords();
} catch (Exception e) {
log.warn("Time series rolling arguments supports only numeric values.");
// throw new IllegalArgumentException("Time series rolling arguments supports only numeric values.");
} }
pollFirstEntryIfNeeded();
} }
private void addTsRecord(Long ts, double value) { private void addTsRecord(Long ts, double value) {
tsRecords.put(ts, value); tsRecords.put(ts, value);
pollFirstEntryIfNeeded(); cleanupExpiredRecords();
} }
private void pollFirstEntryIfNeeded() { private void cleanupExpiredRecords() {
if (tsRecords.size() > MAX_ROLLING_ARGUMENT_ENTRY_SIZE) { if (tsRecords.size() > limit) {
tsRecords.pollFirstEntry(); tsRecords.pollFirstEntry();
} }
tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - timeWindow);
} }
} }

144
application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java

@ -0,0 +1,144 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.utils;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.BasicKvEntry;
import org.thingsboard.server.common.util.KvProtoUtil;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto;
import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto;
import org.thingsboard.server.gen.transport.TransportProtos.TsDoubleValProto;
import org.thingsboard.server.gen.transport.TransportProtos.TsRollingArgumentProto;
import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto;
import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState;
import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState;
import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState;
import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry;
import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry;
import java.util.Optional;
import java.util.TreeMap;
import java.util.UUID;
public class CalculatedFieldUtils {
public static CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) {
return CalculatedFieldEntityCtxIdProto.newBuilder()
.setTenantIdMSB(ctxId.tenantId().getId().getMostSignificantBits())
.setTenantIdLSB(ctxId.tenantId().getId().getLeastSignificantBits())
.setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits())
.setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits())
.setEntityType(ctxId.entityId().getEntityType().name())
.setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits())
.setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits())
.build();
}
public static CalculatedFieldEntityCtxId fromProto(CalculatedFieldEntityCtxIdProto ctxIdProto) {
TenantId tenantId = TenantId.fromUUID(new UUID(ctxIdProto.getTenantIdMSB(), ctxIdProto.getTenantIdLSB()));
EntityId entityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB()));
CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB()));
return new CalculatedFieldEntityCtxId(tenantId, calculatedFieldId, entityId);
}
public static CalculatedFieldStateProto toProto(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state) {
CalculatedFieldStateProto.Builder builder = CalculatedFieldStateProto.newBuilder()
.setId(toProto(stateId))
.setType(state.getType().name());
state.getArguments().forEach((argName, argEntry) -> {
if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) {
builder.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry));
} else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) {
builder.addRollingValueArguments(toRollingArgumentProto(argName, rollingArgumentEntry));
}
});
return builder.build();
}
public static SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) {
SingleValueArgumentProto.Builder builder = SingleValueArgumentProto.newBuilder()
.setArgName(argName);
if (entry.getKvEntryValue() != null) {
builder.setValue(KvProtoUtil.toTsValueProto(entry.getTs(), entry.getKvEntryValue()));
}
Optional.ofNullable(entry.getVersion()).ifPresent(builder::setVersion);
return builder.build();
}
public static TsRollingArgumentProto toRollingArgumentProto(String argName, TsRollingArgumentEntry entry) {
TsRollingArgumentProto.Builder builder = TsRollingArgumentProto.newBuilder()
.setKey(argName)
.setLimit(entry.getLimit())
.setTimeWindow(entry.getTimeWindow());
entry.getTsRecords().forEach((ts, value) -> builder.addTsValue(TsDoubleValProto.newBuilder().setTs(ts).setValue(value).build()));
return builder.build();
}
public static CalculatedFieldState fromProto(CalculatedFieldStateProto proto) {
if (StringUtils.isEmpty(proto.getType())) {
return null;
}
CalculatedFieldType type = CalculatedFieldType.valueOf(proto.getType());
CalculatedFieldState state = switch (type) {
case SIMPLE -> new SimpleCalculatedFieldState();
case SCRIPT -> new ScriptCalculatedFieldState();
};
proto.getSingleValueArgumentsList().forEach(argProto ->
state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto)));
if (CalculatedFieldType.SCRIPT.equals(type)) {
proto.getRollingValueArgumentsList().forEach(argProto ->
state.getArguments().put(argProto.getKey(), fromRollingArgumentProto(argProto)));
}
return state;
}
public static SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) {
if (!proto.hasValue()) {
return new SingleValueArgumentEntry();
}
TsValueProto tsValueProto = proto.getValue();
return new SingleValueArgumentEntry(
tsValueProto.getTs(),
(BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto),
proto.getVersion()
);
}
public static TsRollingArgumentEntry fromRollingArgumentProto(TsRollingArgumentProto proto) {
TreeMap<Long, Double> tsRecords = new TreeMap<>();
proto.getTsValueList().forEach(tsValueProto -> tsRecords.put(tsValueProto.getTs(), tsValueProto.getValue()));
return new TsRollingArgumentEntry(tsRecords, proto.getLimit(), proto.getTimeWindow());
}
}

36
application/src/main/java/org/thingsboard/server/utils/DebugModeRateLimitsConfig.java

@ -0,0 +1,36 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.utils;
import lombok.Data;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
@Data
public class DebugModeRateLimitsConfig {
@Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.enabled:true}")
private boolean ruleChainDebugPerTenantLimitsEnabled;
@Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.configuration:50000:3600}")
private String ruleChainDebugPerTenantLimitsConfiguration;
@Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.enabled:true}")
private boolean calculatedFieldDebugPerTenantLimitsEnabled;
@Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.configuration:50000:3600}")
private String calculatedFieldDebugPerTenantLimitsConfiguration;
}

4
application/src/main/resources/thingsboard.yml

@ -509,9 +509,9 @@ actors:
calculated_fields: calculated_fields:
debug_mode_rate_limits_per_tenant: debug_mode_rate_limits_per_tenant:
# Enable/Disable the rate limit of persisted debug events for all calculated fields per tenant # Enable/Disable the rate limit of persisted debug events for all calculated fields per tenant
enabled: "${ACTORS_CF_DEBUG_MODE_RATE_LIMITS_PER_TENANT_ENABLED:true}" enabled: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_ENABLED:true}"
# The value of DEBUG mode rate limit. By default, no more than 50 thousand events per hour # The value of DEBUG mode rate limit. By default, no more than 50 thousand events per hour
configuration: "${ACTORS_CF_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" configuration: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}"
debug: debug:
settings: settings:

284
application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java

@ -40,8 +40,10 @@ import org.thingsboard.server.controller.CalculatedFieldControllerTest;
import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.service.DaoSqlTest;
import java.util.Map; import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.awaitility.Awaitility.await;
@DaoSqlTest @DaoSqlTest
public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTest { public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTest {
@ -84,20 +86,22 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
// create CF -> perform initial calculation // create CF -> perform initial calculation
CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class);
Thread.sleep(300); await().atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> {
ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp");
assertThat(fahrenheitTemp).isNotNull(); assertThat(fahrenheitTemp).isNotNull();
assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("77.0"); assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("77.0");
});
// update telemetry -> recalculate state // update telemetry -> recalculate state
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}"));
Thread.sleep(300); await().atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> {
fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp");
assertThat(fahrenheitTemp).isNotNull(); assertThat(fahrenheitTemp).isNotNull();
assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0");
});
// update CF output -> perform calculation with updated output // update CF output -> perform calculation with updated output
Output savedOutput = savedCalculatedField.getConfiguration().getOutput(); Output savedOutput = savedCalculatedField.getConfiguration().getOutput();
@ -106,32 +110,35 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
savedOutput.setName("temperatureF"); savedOutput.setName("temperatureF");
savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class);
Thread.sleep(300); await().atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> {
ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF");
assertThat(temperatureF).isNotNull(); assertThat(temperatureF).isNotNull();
assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("86.0"); assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("86.0");
});
// update CF argument -> perform calculation with new argument // update CF argument -> perform calculation with new argument
Argument savedArgument = savedCalculatedField.getConfiguration().getArguments().get("T"); Argument savedArgument = savedCalculatedField.getConfiguration().getArguments().get("T");
savedArgument.setRefEntityKey(new ReferencedEntityKey("deviceTemperature", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); savedArgument.setRefEntityKey(new ReferencedEntityKey("deviceTemperature", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE));
savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class);
Thread.sleep(300); await().atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> {
temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF");
assertThat(temperatureF).isNotNull(); assertThat(temperatureF).isNotNull();
assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0");
});
// update CF expression -> perform calculation with new expression // update CF expression -> perform calculation with new expression
savedCalculatedField.getConfiguration().setExpression("1.8 * T + 32"); savedCalculatedField.getConfiguration().setExpression("1.8 * T + 32");
savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class);
Thread.sleep(300); await().atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> {
temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF");
assertThat(temperatureF).isNotNull(); assertThat(temperatureF).isNotNull();
assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0");
});
} }
@Test @Test
@ -164,20 +171,22 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
// create CF -> state is not ready -> no calculation performed // create CF -> state is not ready -> no calculation performed
CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class);
Thread.sleep(300); await().atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> {
ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp");
assertThat(fahrenheitTemp).isNotNull(); assertThat(fahrenheitTemp).isNotNull();
assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue();
});
// update telemetry -> perform calculation // update telemetry -> perform calculation
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}"));
Thread.sleep(300); await().atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> {
fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp");
assertThat(fahrenheitTemp).isNotNull(); assertThat(fahrenheitTemp).isNotNull();
assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0");
});
} }
@Test @Test
@ -211,20 +220,22 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
// create CF -> perform initial calculation with default value // create CF -> perform initial calculation with default value
CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class);
Thread.sleep(300); await().atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> {
ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp");
assertThat(fahrenheitTemp).isNotNull(); assertThat(fahrenheitTemp).isNotNull();
assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("53.6"); assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("53.6");
});
// update telemetry -> recalculate state // update telemetry -> recalculate state
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}"));
Thread.sleep(300); await().atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> {
fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp");
assertThat(fahrenheitTemp).isNotNull(); assertThat(fahrenheitTemp).isNotNull();
assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0");
});
} }
@Test @Test
@ -275,93 +286,100 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
// create CF and perform initial calculation // create CF and perform initial calculation
doPost("/api/calculatedField", calculatedField, CalculatedField.class); doPost("/api/calculatedField", calculatedField, CalculatedField.class);
Thread.sleep(300); await().atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> {
// result of asset 1 // result of asset 1
ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); ArrayNode z1 = getServerAttributes(asset1.getId(), "z");
assertThat(z1).isNotNull(); assertThat(z1).isNotNull();
assertThat(z1.get(0).get("value").asText()).isEqualTo("51.0"); assertThat(z1.get(0).get("value").asText()).isEqualTo("51.0");
// result of asset 2 // result of asset 2
ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); ArrayNode z2 = getServerAttributes(asset2.getId(), "z");
assertThat(z2).isNotNull(); assertThat(z2).isNotNull();
assertThat(z2.get(0).get("value").asText()).isEqualTo("52.0"); assertThat(z2.get(0).get("value").asText()).isEqualTo("52.0");
});
// update device telemetry -> recalculate state for all assets // update device telemetry -> recalculate state for all assets
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":25}")); doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":25}"));
Thread.sleep(300); await().atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> {
// result of asset 1
ArrayNode z1 = getServerAttributes(asset1.getId(), "z");
assertThat(z1).isNotNull();
assertThat(z1.get(0).get("value").asText()).isEqualTo("36.0");
// result of asset 1 // result of asset 2
z1 = getServerAttributes(asset1.getId(), "z"); ArrayNode z2 = getServerAttributes(asset2.getId(), "z");
assertThat(z1).isNotNull(); assertThat(z2).isNotNull();
assertThat(z1.get(0).get("value").asText()).isEqualTo("36.0"); assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0");
});
// result of asset 2
z2 = getServerAttributes(asset2.getId(), "z");
assertThat(z2).isNotNull();
assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0");
// update asset 1 telemetry -> recalculate state only for asset 1 // update asset 1 telemetry -> recalculate state only for asset 1
doPost("/api/plugins/telemetry/ASSET/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":15}")); doPost("/api/plugins/telemetry/ASSET/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":15}"));
Thread.sleep(300); await().atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> {
// result of asset 1 // result of asset 1
z1 = getServerAttributes(asset1.getId(), "z"); ArrayNode z1 = getServerAttributes(asset1.getId(), "z");
assertThat(z1).isNotNull(); assertThat(z1).isNotNull();
assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0");
// result of asset 2 (no changes) // result of asset 2 (no changes)
z2 = getServerAttributes(asset2.getId(), "z"); ArrayNode z2 = getServerAttributes(asset2.getId(), "z");
assertThat(z2).isNotNull(); assertThat(z2).isNotNull();
assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0");
});
// update asset 2 telemetry -> recalculate state only for asset 2 // update asset 2 telemetry -> recalculate state only for asset 2
doPost("/api/plugins/telemetry/ASSET/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":5}")); doPost("/api/plugins/telemetry/ASSET/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":5}"));
Thread.sleep(300); await().atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> {
// result of asset 1 (no changes) // result of asset 1 (no changes)
z1 = getServerAttributes(asset1.getId(), "z"); ArrayNode z1 = getServerAttributes(asset1.getId(), "z");
assertThat(z1).isNotNull(); assertThat(z1).isNotNull();
assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0");
// result of asset 2 // result of asset 2
z2 = getServerAttributes(asset2.getId(), "z"); ArrayNode z2 = getServerAttributes(asset2.getId(), "z");
assertThat(z2).isNotNull(); assertThat(z2).isNotNull();
assertThat(z2.get(0).get("value").asText()).isEqualTo("30.0"); assertThat(z2.get(0).get("value").asText()).isEqualTo("30.0");
});
// add new entity to profile -> calculate state for new entity // add new entity to profile -> calculate state for new entity
Asset asset3 = createAsset("Test asset 3", assetProfile.getId()); Asset asset3 = createAsset("Test asset 3", assetProfile.getId());
doPost("/api/plugins/telemetry/ASSET/" + asset3.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":13}")); doPost("/api/plugins/telemetry/ASSET/" + asset3.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":13}"));
Thread.sleep(300); Asset finalAsset3 = asset3;
await().atMost(5, TimeUnit.SECONDS)
// result of asset 3 .untilAsserted(() -> {
ArrayNode z3 = getServerAttributes(asset3.getId(), "z"); // result of asset 3
assertThat(z3).isNotNull(); ArrayNode z3 = getServerAttributes(finalAsset3.getId(), "z");
assertThat(z3.get(0).get("value").asText()).isEqualTo("38.0"); assertThat(z3).isNotNull();
assertThat(z3.get(0).get("value").asText()).isEqualTo("38.0");
});
// update device telemetry -> recalculate state for all assets // update device telemetry -> recalculate state for all assets
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":20}")); doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":20}"));
Thread.sleep(300); await().atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> {
// result of asset 1 // result of asset 1
z1 = getServerAttributes(asset1.getId(), "z"); ArrayNode z1 = getServerAttributes(asset1.getId(), "z");
assertThat(z1).isNotNull(); assertThat(z1).isNotNull();
assertThat(z1.get(0).get("value").asText()).isEqualTo("35.0"); assertThat(z1.get(0).get("value").asText()).isEqualTo("35.0");
// result of asset 2 // result of asset 2
z2 = getServerAttributes(asset2.getId(), "z"); ArrayNode z2 = getServerAttributes(asset2.getId(), "z");
assertThat(z2).isNotNull(); assertThat(z2).isNotNull();
assertThat(z2.get(0).get("value").asText()).isEqualTo("25.0"); assertThat(z2.get(0).get("value").asText()).isEqualTo("25.0");
// result of asset 3 // result of asset 3
z3 = getServerAttributes(asset3.getId(), "z"); ArrayNode z3 = getServerAttributes(finalAsset3.getId(), "z");
assertThat(z3).isNotNull(); assertThat(z3).isNotNull();
assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0");
});
// update profile for asset 3 -> delete state for asset 3 // update profile for asset 3 -> delete state for asset 3
AssetProfile newAssetProfile = doPost("/api/assetProfile", createAssetProfile("New Asset Profile"), AssetProfile.class); AssetProfile newAssetProfile = doPost("/api/assetProfile", createAssetProfile("New Asset Profile"), AssetProfile.class);
@ -371,22 +389,24 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
// update device telemetry -> recalculate state for asset 1 and asset 2 // update device telemetry -> recalculate state for asset 1 and asset 2
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":15}")); doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":15}"));
Thread.sleep(300); Asset updatedAsset3 = asset3;
await().atMost(5, TimeUnit.SECONDS)
// result of asset 1 .untilAsserted(() -> {
z1 = getServerAttributes(asset1.getId(), "z"); // result of asset 1
assertThat(z1).isNotNull(); ArrayNode z1 = getServerAttributes(asset1.getId(), "z");
assertThat(z1.get(0).get("value").asText()).isEqualTo("30.0"); assertThat(z1).isNotNull();
assertThat(z1.get(0).get("value").asText()).isEqualTo("30.0");
// result of asset 2
z2 = getServerAttributes(asset2.getId(), "z"); // result of asset 2
assertThat(z2).isNotNull(); ArrayNode z2 = getServerAttributes(asset2.getId(), "z");
assertThat(z2.get(0).get("value").asText()).isEqualTo("20.0"); assertThat(z2).isNotNull();
assertThat(z2.get(0).get("value").asText()).isEqualTo("20.0");
// no changes for asset 3
z3 = getServerAttributes(asset3.getId(), "z"); // no changes for asset 3
assertThat(z3).isNotNull(); ArrayNode z3 = getServerAttributes(updatedAsset3.getId(), "z");
assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); assertThat(z3).isNotNull();
assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0");
});
} }
@Test @Test
@ -421,20 +441,22 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes
// create CF -> ctx is not initialized -> no calculation perform // create CF -> ctx is not initialized -> no calculation perform
CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class);
Thread.sleep(300); await().atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> {
ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp");
assertThat(fahrenheitTemp).isNotNull(); assertThat(fahrenheitTemp).isNotNull();
assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue();
});
// update telemetry -> ctx is not initialized -> no calculation perform // update telemetry -> ctx is not initialized -> no calculation perform
doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}"));
Thread.sleep(300); await().atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> {
fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp");
assertThat(fahrenheitTemp).isNotNull(); assertThat(fahrenheitTemp).isNotNull();
assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue();
});
} }
private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception {

56
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java

@ -20,6 +20,7 @@ import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.MockBean;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.script.api.tbel.DefaultTbelInvokeService; import org.thingsboard.script.api.tbel.DefaultTbelInvokeService;
import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.script.api.tbel.TbelInvokeService;
import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.AttributeScope;
@ -35,7 +36,7 @@ import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedField
import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry;
import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry;
import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.dao.usagerecord.ApiLimitService;
import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.CalculatedFieldResult;
@ -57,7 +58,7 @@ public class ScriptCalculatedFieldStateTest {
private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb"));
private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76"));
private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new LongDataEntry("assetHumidity", 43L), 122L); private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new DoubleDataEntry("assetHumidity", 43.0), 122L);
private final TsRollingArgumentEntry deviceTemperatureArgEntry = createRollingArgEntry(); private final TsRollingArgumentEntry deviceTemperatureArgEntry = createRollingArgEntry();
private final long ts = System.currentTimeMillis(); private final long ts = System.currentTimeMillis();
@ -127,52 +128,7 @@ public class ScriptCalculatedFieldStateTest {
Output output = getCalculatedFieldConfig().getOutput(); Output output = getCalculatedFieldConfig().getOutput();
assertThat(result.getType()).isEqualTo(output.getType()); assertThat(result.getType()).isEqualTo(output.getType());
assertThat(result.getScope()).isEqualTo(output.getScope()); assertThat(result.getScope()).isEqualTo(output.getScope());
assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 13.0, "assetHumidity", 43L)); assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("maxDeviceTemperature", 17.0, "assetHumidity", 43.0)));
}
@Test
void testPerformCalculationWhenOldTelemetry() throws ExecutionException, InterruptedException {
TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry();
TreeMap<Long, Double> values = new TreeMap<>();
values.put(ts - 40000, 4.0);// will not be used for calculation
values.put(ts - 45000, 2.0);// will not be used for calculation
values.put(ts - 20, 0.0);
argumentEntry.setTsRecords(values);
state.arguments = new HashMap<>(Map.of("deviceTemperature", argumentEntry, "assetHumidity", assetHumidityArgEntry));
CalculatedFieldResult result = state.performCalculation(ctx).get();
assertThat(result).isNotNull();
Output output = getCalculatedFieldConfig().getOutput();
assertThat(result.getType()).isEqualTo(output.getType());
assertThat(result.getScope()).isEqualTo(output.getScope());
assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43L));
}
@Test
void testPerformCalculationWhenArgumentsMoreThanLimit() throws ExecutionException, InterruptedException {
TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry();
TreeMap<Long, Double> values = new TreeMap<>();
values.put(ts - 20, 1000.0);// will not be used
values.put(ts - 18, 0.0);
values.put(ts - 16, 0.0);
values.put(ts - 14, 0.0);
values.put(ts - 12, 0.0);
values.put(ts - 10, 0.0);
argumentEntry.setTsRecords(values);
state.arguments = new HashMap<>(Map.of("deviceTemperature", argumentEntry, "assetHumidity", assetHumidityArgEntry));
CalculatedFieldResult result = state.performCalculation(ctx).get();
assertThat(result).isNotNull();
Output output = getCalculatedFieldConfig().getOutput();
assertThat(result.getType()).isEqualTo(output.getType());
assertThat(result.getScope()).isEqualTo(output.getScope());
assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43L));
} }
@Test @Test
@ -189,7 +145,7 @@ public class ScriptCalculatedFieldStateTest {
@Test @Test
void testIsReadyWhenEmptyEntryPresents() { void testIsReadyWhenEmptyEntryPresents() {
state.arguments = new HashMap<>(Map.of("deviceTemperature", TsRollingArgumentEntry.EMPTY, "assetHumidity", assetHumidityArgEntry)); state.arguments = new HashMap<>(Map.of("deviceTemperature", new TsRollingArgumentEntry(5, 30000L), "assetHumidity", assetHumidityArgEntry));
assertThat(state.isReady()).isFalse(); assertThat(state.isReady()).isFalse();
} }
@ -235,7 +191,7 @@ public class ScriptCalculatedFieldStateTest {
config.setArguments(Map.of("deviceTemperature", argument1, "assetHumidity", argument2)); config.setArguments(Map.of("deviceTemperature", argument1, "assetHumidity", argument2));
config.setExpression("var result = 0; foreach(element : deviceTemperature.entrySet()) { result += element.getValue(); } var map = {}; map.put(\"averageDeviceTemperature\", result / deviceTemperature.size()); map.put(\"assetHumidity\", assetHumidity); return map;"); config.setExpression("return {\"maxDeviceTemperature\": deviceTemperature.max(), \"assetHumidity\": assetHumidity.value}");
Output output = new Output(); Output output = new Output();
output.setType(OutputType.ATTRIBUTES); output.setType(OutputType.ATTRIBUTES);

7
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java

@ -20,6 +20,7 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoExtension;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.CalculatedFieldType;
@ -117,7 +118,7 @@ public class SimpleCalculatedFieldStateTest {
"key2", key2ArgEntry "key2", key2ArgEntry
)); ));
Map<String, ArgumentEntry> newArgs = Map.of("key3", TsRollingArgumentEntry.EMPTY); Map<String, ArgumentEntry> newArgs = Map.of("key3", new TsRollingArgumentEntry(10, 30000L));
assertThatThrownBy(() -> state.updateState(newArgs)) assertThatThrownBy(() -> state.updateState(newArgs))
.isInstanceOf(IllegalArgumentException.class) .isInstanceOf(IllegalArgumentException.class)
.hasMessage("Rolling argument entry is not supported for simple calculated fields."); .hasMessage("Rolling argument entry is not supported for simple calculated fields.");
@ -137,7 +138,7 @@ public class SimpleCalculatedFieldStateTest {
Output output = getCalculatedFieldConfig().getOutput(); Output output = getCalculatedFieldConfig().getOutput();
assertThat(result.getType()).isEqualTo(output.getType()); assertThat(result.getType()).isEqualTo(output.getType());
assertThat(result.getScope()).isEqualTo(output.getScope()); assertThat(result.getScope()).isEqualTo(output.getScope());
assertThat(result.getResultMap()).isEqualTo(Map.of("output", 49.0)); assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 49.0)));
} }
@Test @Test
@ -175,7 +176,7 @@ public class SimpleCalculatedFieldStateTest {
"key1", key1ArgEntry, "key1", key1ArgEntry,
"key2", key2ArgEntry "key2", key2ArgEntry
)); ));
state.getArguments().put("key3", SingleValueArgumentEntry.EMPTY); state.getArguments().put("key3", new SingleValueArgumentEntry());
assertThat(state.isReady()).isFalse(); assertThat(state.isReady()).isFalse();
} }

2
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java

@ -40,7 +40,7 @@ public class SingleValueArgumentEntryTest {
@Test @Test
void testUpdateEntryWhenRollingEntryPassed() { void testUpdateEntryWhenRollingEntryPassed() {
assertThatThrownBy(() -> entry.updateEntry(TsRollingArgumentEntry.EMPTY)) assertThatThrownBy(() -> entry.updateEntry(new TsRollingArgumentEntry(5, 30000L)))
.isInstanceOf(IllegalArgumentException.class) .isInstanceOf(IllegalArgumentException.class)
.hasMessage("Unsupported argument entry type for single value argument entry: " + ArgumentEntryType.TS_ROLLING); .hasMessage("Unsupported argument entry type for single value argument entry: " + ArgumentEntryType.TS_ROLLING);
} }

60
application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java

@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf.ctx.state;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.thingsboard.server.common.data.kv.BasicKvEntry;
import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry;
import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry;
@ -40,7 +39,7 @@ public class TsRollingArgumentEntryTest {
values.put(ts - 30, 12.0); values.put(ts - 30, 12.0);
values.put(ts - 20, 17.0); values.put(ts - 20, 17.0);
entry = new TsRollingArgumentEntry(values); entry = new TsRollingArgumentEntry(5, 30000L, values);
} }
@Test @Test
@ -57,18 +56,10 @@ public class TsRollingArgumentEntryTest {
assertThat(entry.getTsRecords().get(ts - 10)).isEqualTo(23.0); assertThat(entry.getTsRecords().get(ts - 10)).isEqualTo(23.0);
} }
@Test
void testUpdateEntryWhenSingleValueEntryWithTheSameTsPassed() {
SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 20, new DoubleDataEntry("key", 23.0), 123L);
assertThat(entry.updateEntry(newEntry)).isFalse();
}
@Test @Test
void testUpdateEntryWhenRollingEntryPassed() { void testUpdateEntryWhenRollingEntryPassed() {
TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry();
TreeMap<Long, Double> values = new TreeMap<>(); TreeMap<Long, Double> values = new TreeMap<>();
values.put(ts - 20, 16.0);
values.put(ts - 10, 7.0); values.put(ts - 10, 7.0);
values.put(ts - 5, 1.0); values.put(ts - 5, 1.0);
newEntry.setTsRecords(values); newEntry.setTsRecords(values);
@ -76,11 +67,11 @@ public class TsRollingArgumentEntryTest {
assertThat(entry.updateEntry(newEntry)).isTrue(); assertThat(entry.updateEntry(newEntry)).isTrue();
assertThat(entry.getTsRecords()).hasSize(5); assertThat(entry.getTsRecords()).hasSize(5);
assertThat(entry.getTsRecords()).isEqualTo(Map.of( assertThat(entry.getTsRecords()).isEqualTo(Map.of(
ts - 40, new DoubleDataEntry("key", 10.0), ts - 40, 10.0,
ts - 30, new DoubleDataEntry("key", 12.0), ts - 30, 12.0,
ts - 20, new DoubleDataEntry("key", 17.0), ts - 20, 17.0,
ts - 10, new DoubleDataEntry("key", 7.0), ts - 10, 7.0,
ts - 5, new DoubleDataEntry("key", 1.0) ts - 5, 1.0
)); ));
} }
@ -90,7 +81,44 @@ public class TsRollingArgumentEntryTest {
assertThatThrownBy(() -> entry.updateEntry(newEntry)) assertThatThrownBy(() -> entry.updateEntry(newEntry))
.isInstanceOf(IllegalArgumentException.class) .isInstanceOf(IllegalArgumentException.class)
.hasMessage("Argument type " + ArgumentEntryType.TS_ROLLING + " only supports numeric values."); .hasMessage("Time series rolling arguments supports only numeric values.");
}
@Test
void testUpdateEntryWhenOldTelemetry() {
TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry();
TreeMap<Long, Double> values = new TreeMap<>();
values.put(ts - 40000, 4.0);// will not be used for calculation
values.put(ts - 45000, 2.0);// will not be used for calculation
values.put(ts - 5, 0.0);
newEntry.setTsRecords(values);
entry = new TsRollingArgumentEntry(3, 30000L);
assertThat(entry.updateEntry(newEntry)).isTrue();
assertThat(entry.getTsRecords()).hasSize(1);
assertThat(entry.getTsRecords()).isEqualTo(Map.of(
ts - 5, 0.0
));
}
@Test
void testPerformCalculationWhenArgumentsMoreThanLimit() {
TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry();
TreeMap<Long, Double> values = new TreeMap<>();
values.put(ts - 20, 1000.0);// will not be used
values.put(ts - 18, 0.0);
values.put(ts - 16, 0.0);
values.put(ts - 14, 0.0);
newEntry.setTsRecords(values);
entry = new TsRollingArgumentEntry(3, 30000L);
assertThat(entry.updateEntry(newEntry)).isTrue();
assertThat(entry.getTsRecords()).hasSize(3);
assertThat(entry.getTsRecords()).isEqualTo(Map.of(
ts - 18, 0.0,
ts - 16, 0.0,
ts - 14, 0.0
));
} }
} }

1
common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java

@ -34,4 +34,5 @@ public class SystemParams {
boolean mobileQrEnabled; boolean mobileQrEnabled;
int maxDebugModeDurationMinutes; int maxDebugModeDurationMinutes;
String ruleChainDebugPerTenantLimitsConfiguration; String ruleChainDebugPerTenantLimitsConfiguration;
String calculatedFieldDebugPerTenantLimitsConfiguration;
} }

1
common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java

@ -91,5 +91,4 @@ public class CalculatedFieldDebugEvent extends Event {
return eventInfo; return eventInfo;
} }
} }

14
common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java

@ -25,8 +25,6 @@ import org.thingsboard.server.common.data.StringUtils;
@Schema @Schema
public class CalculatedFieldDebugEventFilter extends DebugEventFilter { public class CalculatedFieldDebugEventFilter extends DebugEventFilter {
@Schema(description = "String value representing the calculated field id in the event body", example = "ccbfa2fe-c8f5-45d8-bb37-6b61a6e02833")
protected String calculatedFieldId;
@Schema(description = "String value representing the entity id in the event body", example = "57b6bafe-d600-423c-9267-fe31e5218986") @Schema(description = "String value representing the entity id in the event body", example = "57b6bafe-d600-423c-9267-fe31e5218986")
protected String entityId; protected String entityId;
@Schema(description = "String value representing the entity type", allowableValues = "DEVICE") @Schema(description = "String value representing the entity type", allowableValues = "DEVICE")
@ -35,6 +33,13 @@ public class CalculatedFieldDebugEventFilter extends DebugEventFilter {
protected String msgId; protected String msgId;
@Schema(description = "String value representing the message type", example = "POST_TELEMETRY_REQUEST") @Schema(description = "String value representing the message type", example = "POST_TELEMETRY_REQUEST")
protected String msgType; protected String msgType;
@Schema(description = "String value representing the arguments that were used in the calculation performed",
example = "{\"x\":{\"ts\":1739432016629,\"value\":20},\"y\":{\"ts\":1739429717656,\"value\":12}}")
protected String arguments;
@Schema(description = "String value representing the result of a calculation",
example = "{\"x + y\":54}")
protected String result;
@Override @Override
public EventType getEventType() { public EventType getEventType() {
@ -43,8 +48,9 @@ public class CalculatedFieldDebugEventFilter extends DebugEventFilter {
@Override @Override
public boolean isNotEmpty() { public boolean isNotEmpty() {
return super.isNotEmpty() || !StringUtils.isEmpty(calculatedFieldId) || !StringUtils.isEmpty(entityId) return super.isNotEmpty() || !StringUtils.isEmpty(entityId) || !StringUtils.isEmpty(entityType)
|| !StringUtils.isEmpty(entityType) || !StringUtils.isEmpty(msgId) || !StringUtils.isEmpty(msgType); || !StringUtils.isEmpty(msgId) || !StringUtils.isEmpty(msgType)
|| !StringUtils.isEmpty(arguments) || !StringUtils.isEmpty(result);
} }
} }

3
common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java

@ -43,7 +43,8 @@ public enum LimitedApi {
TRANSPORT_MESSAGES_PER_GATEWAY("transport messages per gateway", false), TRANSPORT_MESSAGES_PER_GATEWAY("transport messages per gateway", false),
TRANSPORT_MESSAGES_PER_GATEWAY_DEVICE("transport messages per gateway device", false), TRANSPORT_MESSAGES_PER_GATEWAY_DEVICE("transport messages per gateway device", false),
EMAILS("emails sending", true), EMAILS("emails sending", true),
WS_SUBSCRIPTIONS("WS subscriptions", false); WS_SUBSCRIPTIONS("WS subscriptions", false),
CALCULATED_FIELD_DEBUG_EVENTS(DefaultTenantProfileConfiguration::getCalculatedFieldDebugEventsRateLimit, "calculated field debug events", true);
private Function<DefaultTenantProfileConfiguration, String> configExtractor; private Function<DefaultTenantProfileConfiguration, String> configExtractor;
@Getter @Getter

1
common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java

@ -140,6 +140,7 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura
private long maxDataPointsPerRollingArg; private long maxDataPointsPerRollingArg;
private long maxStateSizeInKBytes; private long maxStateSizeInKBytes;
private long maxSingleValueArgumentSizeInKBytes; private long maxSingleValueArgumentSizeInKBytes;
private String calculatedFieldDebugEventsRateLimit;
@Override @Override
public long getProfileThreshold(ApiUsageRecordKey key) { public long getProfileThreshold(ApiUsageRecordKey key) {

11
common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java

@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.EdgeUtils;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ResourceSubType; import org.thingsboard.server.common.data.ResourceSubType;
import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.TenantProfile;
@ -148,9 +149,13 @@ public class ProtoUtils {
var builder = ComponentLifecycleMsg.builder() var builder = ComponentLifecycleMsg.builder()
.tenantId(TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB()))) .tenantId(TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())))
.entityId(entityId) .entityId(entityId)
.event(ComponentLifecycleEvent.values()[proto.getEventValue()]) .event(ComponentLifecycleEvent.values()[proto.getEventValue()]);
.name(proto.getName()) if (!StringUtils.isEmpty(proto.getName())) {
.oldName(proto.getOldName()); builder.name(proto.getName());
}
if (!StringUtils.isEmpty(proto.getOldName())) {
builder.oldName(proto.getOldName());
}
if (proto.getProfileIdMSB() != 0 || proto.getProfileIdLSB() != 0) { if (proto.getProfileIdMSB() != 0 || proto.getProfileIdLSB() != 0) {
var profileType = EntityType.DEVICE.equals(entityId.getEntityType()) ? EntityType.DEVICE_PROFILE : EntityType.ASSET_PROFILE; var profileType = EntityType.DEVICE.equals(entityId.getEntityType()) ? EntityType.DEVICE_PROFILE : EntityType.ASSET_PROFILE;
builder.profileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB()))); builder.profileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB())));

16
common/proto/src/main/proto/queue.proto

@ -814,11 +814,6 @@ message SingleValueArgumentProto {
int64 version = 3; int64 version = 3;
} }
message CalculatedFieldStateMsgProto {
CalculatedFieldEntityCtxIdProto id = 1;
CalculatedFieldStateProto state = 2;
}
message TsDoubleValProto { message TsDoubleValProto {
int64 ts = 1; int64 ts = 1;
double value = 2; double value = 2;
@ -826,13 +821,16 @@ message TsDoubleValProto {
message TsRollingArgumentProto { message TsRollingArgumentProto {
string key = 1; string key = 1;
repeated TsDoubleValProto tsValue = 2; int32 limit = 2;
int64 timeWindow = 3;
repeated TsDoubleValProto tsValue = 4;
} }
message CalculatedFieldStateProto { message CalculatedFieldStateProto {
string type = 1; CalculatedFieldEntityCtxIdProto id = 1;
repeated SingleValueArgumentProto singleValueArguments = 2; string type = 2;
repeated TsRollingArgumentProto rollingValueArguments = 3; repeated SingleValueArgumentProto singleValueArguments = 3;
repeated TsRollingArgumentProto rollingValueArguments = 4;
} }
//Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level.

6
common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java

@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.js.JsInvokeProtos;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto;
import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.TbQueueRequestTemplate;
@ -160,12 +160,12 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<CalculatedFieldStateMsgProto>> createCalculatedFieldStateConsumer() { public TbQueueConsumer<TbProtoQueueMsg<CalculatedFieldStateProto>> createCalculatedFieldStateConsumer() {
return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getStateTopic()));
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<CalculatedFieldStateMsgProto>> createCalculatedFieldStateProducer() { public TbQueueProducer<TbProtoQueueMsg<CalculatedFieldStateProto>> createCalculatedFieldStateProducer() {
return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getStateTopic()));
} }

12
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java

@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.js.JsInvokeProtos;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto;
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
@ -548,23 +548,23 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<CalculatedFieldStateMsgProto>> createCalculatedFieldStateConsumer() { public TbQueueConsumer<TbProtoQueueMsg<CalculatedFieldStateProto>> createCalculatedFieldStateConsumer() {
return TbKafkaConsumerTemplate.<TbProtoQueueMsg<CalculatedFieldStateMsgProto>>builder() return TbKafkaConsumerTemplate.<TbProtoQueueMsg<CalculatedFieldStateProto>>builder()
.settings(kafkaSettings) .settings(kafkaSettings)
.topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())) .topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic()))
.readFromBeginning(true) .readFromBeginning(true)
.stopWhenRead(true) .stopWhenRead(true)
.clientId("monolith-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()) .clientId("monolith-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet())
.groupId(topicService.buildTopicName("monolith-calculated-field-state-consumer")) .groupId(topicService.buildTopicName("monolith-calculated-field-state-consumer"))
.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), CalculatedFieldStateMsgProto.parseFrom(msg.getData()), msg.getHeaders())) .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), CalculatedFieldStateProto.parseFrom(msg.getData()), msg.getHeaders()))
.admin(cfStateAdmin) .admin(cfStateAdmin)
.statsService(consumerStatsService) .statsService(consumerStatsService)
.build(); .build();
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<CalculatedFieldStateMsgProto>> createCalculatedFieldStateProducer() { public TbQueueProducer<TbProtoQueueMsg<CalculatedFieldStateProto>> createCalculatedFieldStateProducer() {
return TbKafkaProducerTemplate.<TbProtoQueueMsg<CalculatedFieldStateMsgProto>>builder() return TbKafkaProducerTemplate.<TbProtoQueueMsg<CalculatedFieldStateProto>>builder()
.settings(kafkaSettings) .settings(kafkaSettings)
.clientId("monolith-calculated-field-state-" + serviceInfoProvider.getServiceId()) .clientId("monolith-calculated-field-state-" + serviceInfoProvider.getServiceId())
.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())) .defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic()))

12
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java

@ -23,7 +23,7 @@ import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.js.JsInvokeProtos;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto;
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
@ -340,23 +340,23 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory {
} }
@Override @Override
public TbQueueConsumer<TbProtoQueueMsg<CalculatedFieldStateMsgProto>> createCalculatedFieldStateConsumer() { public TbQueueConsumer<TbProtoQueueMsg<CalculatedFieldStateProto>> createCalculatedFieldStateConsumer() {
return TbKafkaConsumerTemplate.<TbProtoQueueMsg<CalculatedFieldStateMsgProto>>builder() return TbKafkaConsumerTemplate.<TbProtoQueueMsg<CalculatedFieldStateProto>>builder()
.settings(kafkaSettings) .settings(kafkaSettings)
.topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())) .topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic()))
.readFromBeginning(true) .readFromBeginning(true)
.stopWhenRead(true) .stopWhenRead(true)
.clientId("tb-rule-engine-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()) .clientId("tb-rule-engine-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet())
.groupId(topicService.buildTopicName("tb-rule-engine-calculated-field-state-consumer")) .groupId(topicService.buildTopicName("tb-rule-engine-calculated-field-state-consumer"))
.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), CalculatedFieldStateMsgProto.parseFrom(msg.getData()), msg.getHeaders())) .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), CalculatedFieldStateProto.parseFrom(msg.getData()), msg.getHeaders()))
.admin(cfStateAdmin) .admin(cfStateAdmin)
.statsService(consumerStatsService) .statsService(consumerStatsService)
.build(); .build();
} }
@Override @Override
public TbQueueProducer<TbProtoQueueMsg<CalculatedFieldStateMsgProto>> createCalculatedFieldStateProducer() { public TbQueueProducer<TbProtoQueueMsg<CalculatedFieldStateProto>> createCalculatedFieldStateProducer() {
return TbKafkaProducerTemplate.<TbProtoQueueMsg<CalculatedFieldStateMsgProto>>builder() return TbKafkaProducerTemplate.<TbProtoQueueMsg<CalculatedFieldStateProto>>builder()
.settings(kafkaSettings) .settings(kafkaSettings)
.clientId("tb-rule-engine-to-calculated-field-state-" + serviceInfoProvider.getServiceId()) .clientId("tb-rule-engine-to-calculated-field-state-" + serviceInfoProvider.getServiceId())
.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())) .defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic()))

6
common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java

@ -17,7 +17,7 @@ package org.thingsboard.server.queue.provider;
import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.js.JsInvokeProtos;
import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto;
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
@ -126,8 +126,8 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory
TbQueueConsumer<TbProtoQueueMsg<ToCalculatedFieldNotificationMsg>> createToCalculatedFieldNotificationsMsgConsumer(); TbQueueConsumer<TbProtoQueueMsg<ToCalculatedFieldNotificationMsg>> createToCalculatedFieldNotificationsMsgConsumer();
TbQueueConsumer<TbProtoQueueMsg<CalculatedFieldStateMsgProto>> createCalculatedFieldStateConsumer(); TbQueueConsumer<TbProtoQueueMsg<CalculatedFieldStateProto>> createCalculatedFieldStateConsumer();
TbQueueProducer<TbProtoQueueMsg<CalculatedFieldStateMsgProto>> createCalculatedFieldStateProducer(); TbQueueProducer<TbProtoQueueMsg<CalculatedFieldStateProto>> createCalculatedFieldStateProducer();
} }

1
common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java

@ -136,6 +136,7 @@ public class DefaultTbelInvokeService extends AbstractScriptInvokeService implem
parserConfig.registerDataType("TbelCfSingleValueArg", TbelCfSingleValueArg.class, TbelCfSingleValueArg::memorySize); parserConfig.registerDataType("TbelCfSingleValueArg", TbelCfSingleValueArg.class, TbelCfSingleValueArg::memorySize);
parserConfig.registerDataType("TbelCfTsRollingArg", TbelCfTsRollingArg.class, TbelCfTsRollingArg::memorySize); parserConfig.registerDataType("TbelCfTsRollingArg", TbelCfTsRollingArg.class, TbelCfTsRollingArg::memorySize);
parserConfig.registerDataType("TbelCfTsDoubleVal", TbelCfTsDoubleVal.class, TbelCfTsDoubleVal::memorySize); parserConfig.registerDataType("TbelCfTsDoubleVal", TbelCfTsDoubleVal.class, TbelCfTsDoubleVal::memorySize);
parserConfig.registerDataType("TbTimeWindow", TbTimeWindow.class, TbTimeWindow::memorySize);
TbUtils.register(parserConfig); TbUtils.register(parserConfig);
executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, "tbel-executor")); executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, "tbel-executor"));
try { try {

36
common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java

@ -0,0 +1,36 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.script.api.tbel;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
public class TbTimeWindow implements TbelCfObject {
public static final long OBJ_SIZE = 32L;
private long startTs;
private long endTs;
private int limit;
@Override
public long memorySize() {
return OBJ_SIZE;
}
}

14
common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java

@ -15,6 +15,20 @@
*/ */
package org.thingsboard.script.api.tbel; package org.thingsboard.script.api.tbel;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
@JsonTypeInfo(
use = JsonTypeInfo.Id.NAME,
include = JsonTypeInfo.As.PROPERTY,
property = "type"
)
@JsonSubTypes({
@JsonSubTypes.Type(value = TbelCfSingleValueArg.class, name = "SINGLE_VALUE"),
@JsonSubTypes.Type(value = TbelCfTsRollingArg.class, name = "TS_ROLLING")
})
public interface TbelCfArg extends TbelCfObject { public interface TbelCfArg extends TbelCfObject {
String getType();
} }

16
common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfSingleValueArg.java

@ -15,6 +15,8 @@
*/ */
package org.thingsboard.script.api.tbel; package org.thingsboard.script.api.tbel;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data; import lombok.Data;
@Data @Data
@ -23,9 +25,23 @@ public class TbelCfSingleValueArg implements TbelCfArg {
private final long ts; private final long ts;
private final Object value; private final Object value;
@JsonCreator
public TbelCfSingleValueArg(
@JsonProperty("ts") long ts,
@JsonProperty("value") Object value
) {
this.ts = ts;
this.value = value;
}
@Override @Override
public long memorySize() { public long memorySize() {
return 8L; // TODO; return 8L; // TODO;
} }
@Override
public String getType() {
return "SINGLE_VALUE";
}
} }

22
common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java

@ -15,7 +15,9 @@
*/ */
package org.thingsboard.script.api.tbel; package org.thingsboard.script.api.tbel;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Getter; import lombok.Getter;
import java.util.Collections; import java.util.Collections;
@ -27,10 +29,23 @@ import static org.thingsboard.script.api.tbel.TbelCfTsDoubleVal.OBJ_SIZE;
public class TbelCfTsRollingArg implements TbelCfArg, Iterable<TbelCfTsDoubleVal> { public class TbelCfTsRollingArg implements TbelCfArg, Iterable<TbelCfTsDoubleVal> {
@Getter
private final TbTimeWindow timeWindow;
@Getter @Getter
private final List<TbelCfTsDoubleVal> values; private final List<TbelCfTsDoubleVal> values;
public TbelCfTsRollingArg(List<TbelCfTsDoubleVal> values) { @JsonCreator
public TbelCfTsRollingArg(
@JsonProperty("timeWindow") TbTimeWindow timeWindow,
@JsonProperty("values") List<TbelCfTsDoubleVal> values
) {
this.timeWindow = timeWindow;
this.values = Collections.unmodifiableList(values);
}
public TbelCfTsRollingArg(int limit, long timeWindow, List<TbelCfTsDoubleVal> values) {
long ts = System.currentTimeMillis();
this.timeWindow = new TbTimeWindow(ts - timeWindow, ts, limit);
this.values = Collections.unmodifiableList(values); this.values = Collections.unmodifiableList(values);
} }
@ -70,4 +85,9 @@ public class TbelCfTsRollingArg implements TbelCfArg, Iterable<TbelCfTsDoubleVal
values.forEach(action); values.forEach(action);
} }
@Override
public String getType() {
return "TS_ROLLING";
}
} }

4
dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java

@ -31,7 +31,7 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.model.BaseEntity; import org.thingsboard.server.dao.model.BaseEntity;
import org.thingsboard.server.dao.model.BaseSqlEntity; import org.thingsboard.server.dao.model.BaseVersionedEntity;
import org.thingsboard.server.dao.util.mapping.JsonConverter; import org.thingsboard.server.dao.util.mapping.JsonConverter;
import java.util.UUID; import java.util.UUID;
@ -52,7 +52,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.DEBUG_SETTINGS;
@EqualsAndHashCode(callSuper = true) @EqualsAndHashCode(callSuper = true)
@Entity @Entity
@Table(name = CALCULATED_FIELD_TABLE_NAME) @Table(name = CALCULATED_FIELD_TABLE_NAME)
public class CalculatedFieldEntity extends BaseSqlEntity<CalculatedField> implements BaseEntity<CalculatedField> { public class CalculatedFieldEntity extends BaseVersionedEntity<CalculatedField> implements BaseEntity<CalculatedField> {
@Column(name = CALCULATED_FIELD_TENANT_ID_COLUMN) @Column(name = CALCULATED_FIELD_TENANT_ID_COLUMN)
private UUID tenantId; private UUID tenantId;

14
dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java

@ -59,6 +59,8 @@ public interface CalculatedFieldDebugEventRepository extends EventRepository<Cal
"AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " +
"AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " +
"AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " +
"AND (:eventArguments IS NULL OR e.e_args ILIKE concat('%', :eventArguments, '%')) " +
"AND (:eventResult IS NULL OR e.e_result ILIKE concat('%', :eventResult, '%')) " +
"AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " +
"AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))"
, ,
@ -73,6 +75,8 @@ public interface CalculatedFieldDebugEventRepository extends EventRepository<Cal
"AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " +
"AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " +
"AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " +
"AND (:eventArguments IS NULL OR e.e_args ILIKE concat('%', :eventArguments, '%')) " +
"AND (:eventResult IS NULL OR e.e_result ILIKE concat('%', :eventResult, '%')) " +
"AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " +
"AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))"
) )
@ -80,12 +84,14 @@ public interface CalculatedFieldDebugEventRepository extends EventRepository<Cal
@Param("entityId") UUID entityId, @Param("entityId") UUID entityId,
@Param("startTime") Long startTime, @Param("startTime") Long startTime,
@Param("endTime") Long endTime, @Param("endTime") Long endTime,
@Param("serviceId") String server, @Param("serviceId") String serviceId,
@Param("calculatedFieldId") UUID calculatedFieldId, @Param("calculatedFieldId") UUID calculatedFieldId,
@Param("eventEntityId") String eventEntityId, @Param("eventEntityId") String eventEntityId,
@Param("eventEntityType") String eventEntityType, @Param("eventEntityType") String eventEntityType,
@Param("msgId") String eventMsgId, @Param("msgId") String eventMsgId,
@Param("msgType") String eventMsgType, @Param("msgType") String eventMsgType,
@Param("eventArguments") String eventArguments,
@Param("eventResult") String eventResult,
@Param("isError") boolean isError, @Param("isError") boolean isError,
@Param("error") String error, @Param("error") String error,
Pageable pageable); Pageable pageable);
@ -117,18 +123,22 @@ public interface CalculatedFieldDebugEventRepository extends EventRepository<Cal
"AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " +
"AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " +
"AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " +
"AND (:eventArguments IS NULL OR e.e_args ILIKE concat('%', :eventArguments, '%')) " +
"AND (:eventResult IS NULL OR e.e_result ILIKE concat('%', :eventResult, '%')) " +
"AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " +
"AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))") "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))")
void removeEvents(@Param("tenantId") UUID tenantId, void removeEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId, @Param("entityId") UUID entityId,
@Param("startTime") Long startTime, @Param("startTime") Long startTime,
@Param("endTime") Long endTime, @Param("endTime") Long endTime,
@Param("serviceId") String server, @Param("serviceId") String serviceId,
@Param("calculatedFieldId") UUID calculatedFieldId, @Param("calculatedFieldId") UUID calculatedFieldId,
@Param("eventEntityId") String eventEntityId, @Param("eventEntityId") String eventEntityId,
@Param("eventEntityType") String eventEntityType, @Param("eventEntityType") String eventEntityType,
@Param("msgId") String eventMsgId, @Param("msgId") String eventMsgId,
@Param("msgType") String eventMsgType, @Param("msgType") String eventMsgType,
@Param("eventArguments") String eventArguments,
@Param("eventResult") String eventResult,
@Param("isError") boolean isError, @Param("isError") boolean isError,
@Param("error") String error); @Param("error") String error);

10
dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java

@ -294,7 +294,6 @@ public class JpaBaseEventDao implements EventDao {
} }
private PageData<? extends Event> findEventByFilter(UUID tenantId, UUID entityId, CalculatedFieldDebugEventFilter eventFilter, TimePageLink pageLink) { private PageData<? extends Event> findEventByFilter(UUID tenantId, UUID entityId, CalculatedFieldDebugEventFilter eventFilter, TimePageLink pageLink) {
parseUUID(eventFilter.getCalculatedFieldId(), "Calculated Field Id");
parseUUID(eventFilter.getEntityId(), "Entity Id"); parseUUID(eventFilter.getEntityId(), "Entity Id");
parseUUID(eventFilter.getMsgId(), "Message Id"); parseUUID(eventFilter.getMsgId(), "Message Id");
return DaoUtil.toPageData( return DaoUtil.toPageData(
@ -304,11 +303,13 @@ public class JpaBaseEventDao implements EventDao {
pageLink.getStartTime(), pageLink.getStartTime(),
pageLink.getEndTime(), pageLink.getEndTime(),
eventFilter.getServer(), eventFilter.getServer(),
UUID.fromString(eventFilter.getCalculatedFieldId()), entityId,
eventFilter.getEntityId(), eventFilter.getEntityId(),
eventFilter.getEntityType(), eventFilter.getEntityType(),
eventFilter.getMsgId(), eventFilter.getMsgId(),
eventFilter.getMsgType(), eventFilter.getMsgType(),
eventFilter.getArguments(),
eventFilter.getResult(),
eventFilter.isError(), eventFilter.isError(),
eventFilter.getErrorStr(), eventFilter.getErrorStr(),
DaoUtil.toPageable(pageLink, EventEntity.eventColumnMap))); DaoUtil.toPageable(pageLink, EventEntity.eventColumnMap)));
@ -389,7 +390,6 @@ public class JpaBaseEventDao implements EventDao {
} }
private void removeEventsByFilter(UUID tenantId, UUID entityId, CalculatedFieldDebugEventFilter eventFilter, Long startTime, Long endTime) { private void removeEventsByFilter(UUID tenantId, UUID entityId, CalculatedFieldDebugEventFilter eventFilter, Long startTime, Long endTime) {
parseUUID(eventFilter.getCalculatedFieldId(), "Calculated Field Id");
parseUUID(eventFilter.getEntityId(), "Entity Id"); parseUUID(eventFilter.getEntityId(), "Entity Id");
parseUUID(eventFilter.getMsgId(), "Message Id"); parseUUID(eventFilter.getMsgId(), "Message Id");
calculatedFieldDebugEventRepository.removeEvents( calculatedFieldDebugEventRepository.removeEvents(
@ -398,11 +398,13 @@ public class JpaBaseEventDao implements EventDao {
startTime, startTime,
endTime, endTime,
eventFilter.getServer(), eventFilter.getServer(),
UUID.fromString(eventFilter.getCalculatedFieldId()), entityId,
eventFilter.getEntityId(), eventFilter.getEntityId(),
eventFilter.getEntityType(), eventFilter.getEntityType(),
eventFilter.getMsgId(), eventFilter.getMsgId(),
eventFilter.getMsgType(), eventFilter.getMsgType(),
eventFilter.getArguments(),
eventFilter.getResult(),
eventFilter.isError(), eventFilter.isError(),
eventFilter.getErrorStr()); eventFilter.getErrorStr());
} }

Loading…
Cancel
Save