diff --git a/application/pom.xml b/application/pom.xml index d56fbe3708..6ce77646f8 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -373,6 +373,10 @@ com.google.firebase firebase-admin + + org.rocksdb + rocksdbjni + diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index 406a288505..ddfac0d768 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -31,9 +31,10 @@ DO $$ || jsonb_build_object( 'processingSettings', jsonb_build_object( 'type', 'ADVANCED', - 'timeseries', jsonb_build_object('type', 'ON_EVERY_MESSAGE'), - 'latest', jsonb_build_object('type', 'SKIP'), - 'webSockets', jsonb_build_object('type', 'ON_EVERY_MESSAGE') + 'timeseries', jsonb_build_object('type', 'ON_EVERY_MESSAGE'), + 'latest', jsonb_build_object('type', 'SKIP'), + 'webSockets', jsonb_build_object('type', 'ON_EVERY_MESSAGE'), + 'calculatedFields', jsonb_build_object('type', 'ON_EVERY_MESSAGE') ) ) )::text, @@ -61,3 +62,5 @@ DO $$ $$; -- UPDATE SAVE TIME SERIES NODES END + +ALTER TABLE api_usage_state ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index eb76473386..50a777840c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -41,13 +41,18 @@ import org.thingsboard.script.api.js.JsInvokeService; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; +import org.thingsboard.server.cache.limits.RateLimitService; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.common.data.event.RuleChainDebugEvent; import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.limit.LimitedApi; +import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; @@ -62,6 +67,7 @@ import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.ClaimDevicesService; @@ -94,6 +100,7 @@ import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TenantProfileService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; @@ -101,6 +108,10 @@ import org.thingsboard.server.queue.discovery.DiscoveryService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; @@ -121,13 +132,17 @@ import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import org.thingsboard.server.service.transport.TbCoreToTransportService; +import org.thingsboard.server.utils.DebugModeRateLimitsConfig; import java.io.PrintWriter; import java.io.StringWriter; +import java.util.Map; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; @Slf4j @Component @@ -156,6 +171,18 @@ public class ActorSystemContext { } }; + private static final FutureCallback CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK = new FutureCallback<>() { + @Override + public void onSuccess(@Nullable Void event) { + + } + + @Override + public void onFailure(Throwable th) { + log.error("Could not save debug Event for Calculated Field", th); + } + }; + private final ConcurrentMap debugPerTenantLimits = new ConcurrentHashMap<>(); public ConcurrentMap getDebugPerTenantLimits() { @@ -289,6 +316,7 @@ public class ActorSystemContext { @Getter private TbEntityViewService tbEntityViewService; + @Lazy @Autowired @Getter private TelemetrySubscriptionService tsSubService; @@ -394,6 +422,10 @@ public class ActorSystemContext { @Getter private SlackService slackService; + @Autowired + @Getter + private CalculatedFieldService calculatedFieldService; + @Lazy @Autowired(required = false) @Getter @@ -416,6 +448,21 @@ public class ActorSystemContext { @Getter private TbCoreToTransportService tbCoreToTransportService; + @Lazy + @Autowired(required = false) + @Getter + private ApiLimitService apiLimitService; + + @Lazy + @Autowired(required = false) + @Getter + private RateLimitService rateLimitService; + + @Lazy + @Autowired(required = false) + @Getter + private DebugModeRateLimitsConfig debugModeRateLimitsConfig; + /** * The following Service will be null if we operate in tb-core mode */ @@ -487,6 +534,21 @@ public class ActorSystemContext { @Getter private EntityService entityService; + @Lazy + @Autowired(required = false) + @Getter + private CalculatedFieldProcessingService calculatedFieldProcessingService; + + @Lazy + @Autowired(required = false) + @Getter + private CalculatedFieldStateService calculatedFieldStateService; + + @Lazy + @Autowired(required = false) + @Getter + private CalculatedFieldEntityProfileCache calculatedFieldEntityProfileCache; + @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter private long maxConcurrentSessionsPerDevice; @@ -558,14 +620,6 @@ public class ActorSystemContext { @Getter private long sessionReportTimeout; - @Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.enabled:true}") - @Getter - private boolean debugPerTenantEnabled; - - @Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") - @Getter - private String debugPerTenantLimitsConfiguration; - @Value("${actors.rpc.submit_strategy:BURST}") @Getter private String rpcSubmitStrategy; @@ -719,9 +773,9 @@ public class ActorSystemContext { } private boolean checkLimits(TenantId tenantId, TbMsg tbMsg, Throwable error) { - if (debugPerTenantEnabled) { + if (debugModeRateLimitsConfig.isRuleChainDebugPerTenantLimitsEnabled()) { DebugTbRateLimits debugTbRateLimits = debugPerTenantLimits.computeIfAbsent(tenantId, id -> - new DebugTbRateLimits(new TbRateLimits(debugPerTenantLimitsConfiguration), false)); + new DebugTbRateLimits(new TbRateLimits(debugModeRateLimitsConfig.getRuleChainDebugPerTenantLimitsConfiguration()), false)); if (!debugTbRateLimits.getTbRateLimits().tryConsume()) { if (!debugTbRateLimits.isRuleChainEventSaved()) { @@ -751,6 +805,51 @@ public class ActorSystemContext { Futures.addCallback(future, RULE_CHAIN_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); } + public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, UUID tbMsgId, TbMsgType tbMsgType, String result, String errorMessage) { + if (checkLimits(tenantId)) { + try { + CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() + .tenantId(tenantId) + .entityId(calculatedFieldId.getId()) + .serviceId(getServiceId()) + .calculatedFieldId(calculatedFieldId) + .eventEntity(entityId); + if (tbMsgId != null) { + eventBuilder.msgId(tbMsgId); + } + if (tbMsgType != null) { + eventBuilder.msgType(tbMsgType.name()); + } + if (arguments != null) { + eventBuilder.arguments(JacksonUtil.toString( + arguments.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toTbelCfArg())) + )); + } + if (result != null) { + eventBuilder.result(result); + } + if (errorMessage != null) { + eventBuilder.error(errorMessage); + } + + ListenableFuture future = eventService.saveAsync(eventBuilder.build()); + Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); + } catch (IllegalArgumentException ex) { + log.warn("Failed to persist calculated field debug message", ex); + } + } + } + + private boolean checkLimits(TenantId tenantId) { + if (debugModeRateLimitsConfig.isCalculatedFieldDebugPerTenantLimitsEnabled() && + !rateLimitService.checkRateLimit(LimitedApi.CALCULATED_FIELD_DEBUG_EVENTS, (Object) tenantId, debugModeRateLimitsConfig.getCalculatedFieldDebugPerTenantLimitsConfiguration())) { + log.trace("[{}] Calculated field debug event limits exceeded!", tenantId); + return false; + } + return true; + } + public static Exception toException(Throwable error) { return Exception.class.isInstance(error) ? (Exception) error : new Exception(error); } diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index dc6a3bcf5e..50fa7e2f8d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; @@ -87,6 +88,7 @@ public class AppActor extends ContextAwareActor { case APP_INIT_MSG: break; case PARTITION_CHANGE_MSG: + case CF_PARTITIONS_CHANGE_MSG: ctx.broadcastToChildren(msg, true); break; case COMPONENT_LIFE_CYCLE_MSG: @@ -111,6 +113,18 @@ public class AppActor extends ContextAwareActor { case SESSION_TIMEOUT_MSG: ctx.broadcastToChildrenByType(msg, EntityType.TENANT); break; + case CF_INIT_MSG: + case CF_LINK_INIT_MSG: + case CF_STATE_RESTORE_MSG: + case CF_ENTITY_LIFECYCLE_MSG: + //TODO: use priority from the message body. For example, messages about CF lifecycle are important and Device lifecycle are not. + // same for the Linked telemetry. + onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, true); + break; + case CF_TELEMETRY_MSG: + case CF_LINKED_TELEMETRY_MSG: + onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, false); + break; default: return false; } @@ -175,6 +189,19 @@ public class AppActor extends ContextAwareActor { } } + private void onToCalculatedFieldSystemActorMsg(ToCalculatedFieldSystemMsg msg, boolean priority) { + getOrCreateTenantActor(msg.getTenantId()).ifPresentOrElse(tenantActor -> { + if (priority) { + tenantActor.tellWithHighPriority(msg); + } else { + tenantActor.tell(msg); + } + }, () -> { + msg.getCallback().onSuccess(); + }); + } + + private void onToDeviceActorMsg(TenantAwareMsg msg, boolean priority) { getOrCreateTenantActor(msg.getTenantId()).ifPresentOrElse(tenantActor -> { if (priority) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/AbstractCalculatedFieldActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/AbstractCalculatedFieldActor.java new file mode 100644 index 0000000000..6cf34599de --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/AbstractCalculatedFieldActor.java @@ -0,0 +1,70 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.DebugModeUtil; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.service.ContextAwareActor; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; + +@Slf4j +public abstract class AbstractCalculatedFieldActor extends ContextAwareActor { + + protected final TenantId tenantId; + + public AbstractCalculatedFieldActor(ActorSystemContext systemContext, TenantId tenantId) { + super(systemContext); + this.tenantId = tenantId; + } + + @Override + protected boolean doProcess(TbActorMsg msg) { + if (msg instanceof ToCalculatedFieldSystemMsg cfm) { + Exception cause; + try { + return doProcessCfMsg(cfm); + } catch (CalculatedFieldException cfe) { + if (DebugModeUtil.isDebugFailuresAvailable(cfe.getCtx().getCalculatedField())) { + String message; + if (cfe.getErrorMessage() != null) { + message = cfe.getErrorMessage(); + } else if (cfe.getCause() != null) { + message = cfe.getCause().getMessage(); + } else { + message = "N/A"; + } + systemContext.persistCalculatedFieldDebugEvent(tenantId, cfe.getCtx().getCfId(), cfe.getEventEntity(), cfe.getArguments(), cfe.getMsgId(), cfe.getMsgType(), null, message); + } + cause = cfe.getCause(); + } catch (Exception e) { + logProcessingException(e); + cause = e; + } + cfm.getCallback().onFailure(cause); + return true; + } else { + return false; + } + } + + abstract void logProcessingException(Exception e); + + abstract boolean doProcessCfMsg(ToCalculatedFieldSystemMsg msg) throws CalculatedFieldException; + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java new file mode 100644 index 0000000000..350a5776cf --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -0,0 +1,81 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActorCtx; +import org.thingsboard.server.actors.TbActorException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; + +@Slf4j +public class CalculatedFieldEntityActor extends AbstractCalculatedFieldActor { + + private final CalculatedFieldEntityMessageProcessor processor; + + CalculatedFieldEntityActor(ActorSystemContext systemContext, TenantId tenantId, EntityId entityId) { + super(systemContext, tenantId); + this.processor = new CalculatedFieldEntityMessageProcessor(systemContext, tenantId, entityId); + } + + @Override + public void init(TbActorCtx ctx) throws TbActorException { + super.init(ctx); + log.debug("[{}][{}] Starting CF entity actor.", processor.tenantId, processor.entityId); + try { + processor.init(ctx); + log.debug("[{}][{}] CF entity actor started.", processor.tenantId, processor.entityId); + } catch (Exception e) { + log.warn("[{}][{}] Unknown failure", processor.tenantId, processor.entityId, e); + throw new TbActorException("Failed to initialize CF entity actor", e); + } + } + + @Override + protected boolean doProcessCfMsg(ToCalculatedFieldSystemMsg msg) throws CalculatedFieldException { + switch (msg.getMsgType()) { + case CF_PARTITIONS_CHANGE_MSG: + processor.process((CalculatedFieldPartitionChangeMsg) msg); + break; + case CF_STATE_RESTORE_MSG: + processor.process((CalculatedFieldStateRestoreMsg) msg); + break; + case CF_ENTITY_INIT_CF_MSG: + processor.process((EntityInitCalculatedFieldMsg) msg); + break; + case CF_ENTITY_DELETE_MSG: + processor.process((CalculatedFieldEntityDeleteMsg) msg); + break; + case CF_ENTITY_TELEMETRY_MSG: + processor.process((EntityCalculatedFieldTelemetryMsg) msg); + break; + case CF_LINKED_TELEMETRY_MSG: + processor.process((EntityCalculatedFieldLinkedTelemetryMsg) msg); + break; + default: + return false; + } + return true; + } + + @Override + void logProcessingException(Exception e) { + log.warn("[{}][{}] Processing failure", tenantId, processor.entityId, e); + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java new file mode 100644 index 0000000000..6dc2f26050 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActor; +import org.thingsboard.server.actors.TbActorId; +import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; +import org.thingsboard.server.actors.TbEntityActorId; +import org.thingsboard.server.actors.device.DeviceActor; +import org.thingsboard.server.actors.service.ContextBasedCreator; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +public class CalculatedFieldEntityActorCreator extends ContextBasedCreator { + + private final TenantId tenantId; + private final EntityId entityId; + + public CalculatedFieldEntityActorCreator(ActorSystemContext context, TenantId tenantId, EntityId entityId) { + super(context); + this.tenantId = tenantId; + this.entityId = entityId; + } + + @Override + public TbActorId createActorId() { + return new TbCalculatedFieldEntityActorId(entityId); + } + + @Override + public TbActor createActor() { + return new CalculatedFieldEntityActor(context, tenantId, entityId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityDeleteMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityDeleteMsg.java new file mode 100644 index 0000000000..3ca6e8596a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityDeleteMsg.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; + +@Data +public class CalculatedFieldEntityDeleteMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final TbCallback callback; + + public CalculatedFieldEntityDeleteMsg(TenantId tenantId, + EntityId entityId, + TbCallback callback) { + this.tenantId = tenantId; + this.entityId = entityId; + this.callback = callback; + } + + @Override + public MsgType getMsgType() { + return MsgType.CF_ENTITY_DELETE_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java new file mode 100644 index 0000000000..f7fc204c0f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -0,0 +1,439 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import com.google.common.util.concurrent.ListenableFuture; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.DebugModeUtil; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActorCtx; +import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; +import org.thingsboard.server.service.cf.CalculatedFieldResult; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + + +/** + * @author Andrew Shvayka + */ +@Slf4j +public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareMsgProcessor { + // (1 for result persistence + 1 for the state persistence ) + public static final int CALLBACKS_PER_CF = 2; + + final TenantId tenantId; + final EntityId entityId; + final CalculatedFieldProcessingService cfService; + final CalculatedFieldStateService cfStateService; + final int partition; + + TbActorCtx ctx; + Map states = new HashMap<>(); + + CalculatedFieldEntityMessageProcessor(ActorSystemContext systemContext, TenantId tenantId, EntityId entityId) { + super(systemContext); + this.tenantId = tenantId; + this.entityId = entityId; + this.cfService = systemContext.getCalculatedFieldProcessingService(); + this.cfStateService = systemContext.getCalculatedFieldStateService(); + this.partition = systemContext.getCalculatedFieldEntityProfileCache().getEntityIdPartition(tenantId, entityId); + } + + void init(TbActorCtx ctx) { + this.ctx = ctx; + } + + public void process(CalculatedFieldPartitionChangeMsg msg) { + if (!msg.getPartitions()[partition]) { + log.info("[{}][{}] Stopping entity actor due to change partition event.", partition, entityId); + ctx.stop(ctx.getSelf()); + } + } + + public void process(CalculatedFieldStateRestoreMsg msg) { + CalculatedFieldId cfId = msg.getId().cfId(); + log.info("[{}] [{}] Processing CF state restore msg.", msg.getId().entityId(), cfId); + if (msg.getState() != null) { + states.put(cfId, msg.getState()); + } else { + states.remove(cfId); + } + } + + public void process(EntityInitCalculatedFieldMsg msg) throws CalculatedFieldException { + log.info("[{}] Processing entity init CF msg.", msg.getCtx().getCfId()); + var ctx = msg.getCtx(); + if (msg.isForceReinit()) { + log.info("Force reinitialization of CF: [{}].", ctx.getCfId()); + states.remove(ctx.getCfId()); + } + try { + var state = getOrInitState(ctx); + if (state.isSizeOk()) { + processStateIfReady(ctx, Collections.singletonList(ctx.getCfId()), state, null, null, msg.getCallback()); + } else { + throw new RuntimeException(ctx.getSizeExceedsLimitMessage()); + } + } catch (Exception e) { + if (e instanceof CalculatedFieldException cfe) { + throw cfe; + } + throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); + } + } + + public void process(CalculatedFieldEntityDeleteMsg msg) { + log.info("[{}] Processing CF entity delete msg.", msg.getEntityId()); + if (this.entityId.equals(msg.getEntityId())) { + MultipleTbCallback multipleTbCallback = new MultipleTbCallback(states.size(), msg.getCallback()); + states.forEach((cfId, state) -> cfStateService.removeState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), multipleTbCallback)); + ctx.stop(ctx.getSelf()); + } else { + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + var state = states.remove(cfId); + if (state != null) { + cfStateService.removeState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); + } + } + } + + public void process(EntityCalculatedFieldTelemetryMsg msg) throws CalculatedFieldException { + log.info("[{}] Processing CF telemetry msg.", msg.getEntityId()); + var proto = msg.getProto(); + var numberOfCallbacks = CALLBACKS_PER_CF * (msg.getEntityIdFields().size() + msg.getProfileIdFields().size()); + MultipleTbCallback callback = new MultipleTbCallback(numberOfCallbacks, msg.getCallback()); + List cfIdList = getCalculatedFieldIds(proto); + Set cfIdSet = new HashSet<>(cfIdList); + for (var ctx : msg.getEntityIdFields()) { + process(ctx, proto, cfIdSet, cfIdList, callback); + } + for (var ctx : msg.getProfileIdFields()) { + process(ctx, proto, cfIdSet, cfIdList, callback); + } + } + + public void process(EntityCalculatedFieldLinkedTelemetryMsg msg) throws CalculatedFieldException { + log.info("[{}] Processing CF link telemetry msg.", msg.getEntityId()); + var proto = msg.getProto(); + var ctx = msg.getCtx(); + var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); + try { + List cfIds = getCalculatedFieldIds(proto); + if (cfIds.contains(ctx.getCfId())) { + callback.onSuccess(CALLBACKS_PER_CF); + } else { + if (proto.getTsDataCount() > 0) { + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); + } else if (proto.getAttrDataCount() > 0) { + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); + } else if (proto.getRemovedTsKeysCount() > 0) { + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArgumentsWithFetchedValue(ctx, proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); + } else if (proto.getRemovedAttrKeysCount() > 0) { + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArgumentsWithDefaultValue(ctx, msg.getEntityId(), proto.getScope(), proto.getRemovedAttrKeysList()), toTbMsgId(proto), toTbMsgType(proto)); + } else { + callback.onSuccess(CALLBACKS_PER_CF); + } + } + } catch (Exception e) { + throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); + } + } + + private void process(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, Collection cfIds, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { + try { + if (cfIds.contains(ctx.getCfId())) { + callback.onSuccess(CALLBACKS_PER_CF); + } else { + if (proto.getTsDataCount() > 0) { + processTelemetry(ctx, proto, cfIdList, callback); + } else if (proto.getAttrDataCount() > 0) { + processAttributes(ctx, proto, cfIdList, callback); + } else if (proto.getRemovedTsKeysCount() > 0) { + processRemovedTelemetry(ctx, proto, cfIdList, callback); + } else if (proto.getRemovedAttrKeysCount() > 0) { + processRemovedAttributes(ctx, proto, cfIdList, callback); + } else { + callback.onSuccess(CALLBACKS_PER_CF); + } + } + } catch (Exception e) { + if (e instanceof CalculatedFieldException cfe) { + throw cfe; + } + throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); + } + } + + private void processTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); + } + + private void processAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); + } + + private void processRemovedTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArgumentsWithFetchedValue(ctx, proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); + } + + private void processRemovedAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArgumentsWithDefaultValue(ctx, proto.getScope(), proto.getRemovedAttrKeysList()), toTbMsgId(proto), toTbMsgType(proto)); + } + + private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List cfIdList, MultipleTbCallback callback, + Map newArgValues, UUID tbMsgId, TbMsgType tbMsgType) throws CalculatedFieldException { + if (newArgValues.isEmpty()) { + log.info("[{}] No new argument values to process for CF.", ctx.getCfId()); + callback.onSuccess(CALLBACKS_PER_CF); + } + CalculatedFieldState state = states.get(ctx.getCfId()); + boolean justRestored = false; + if (state == null) { + state = getOrInitState(ctx); + justRestored = true; + } + if (state.isSizeOk()) { + if (state.updateState(newArgValues) || justRestored) { + cfIdList = new ArrayList<>(cfIdList); + cfIdList.add(ctx.getCfId()); + processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback); + } else { + callback.onSuccess(CALLBACKS_PER_CF); + } + } else { + throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(); + } + } + + @SneakyThrows + private CalculatedFieldState getOrInitState(CalculatedFieldCtx ctx) { + CalculatedFieldState state = states.get(ctx.getCfId()); + if (state != null) { + return state; + } else { + ListenableFuture stateFuture = systemContext.getCalculatedFieldProcessingService().fetchStateFromDb(ctx, entityId); + // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. + // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. + // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, + // but this will significantly complicate the code. + state = stateFuture.get(1, TimeUnit.MINUTES); + state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); + states.put(ctx.getCfId(), state); + } + return state; + } + + private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) throws CalculatedFieldException { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); + boolean stateSizeOk; + if (ctx.isInitialized() && state.isReady()) { + try { + CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); + state.checkStateSize(ctxId, ctx.getMaxStateSize()); + stateSizeOk = state.isSizeOk(); + if (stateSizeOk) { + cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); + if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null); + } + } + } catch (Exception e) { + throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).msgId(tbMsgId).msgType(tbMsgType).arguments(state.getArguments()).cause(e).build(); + } + } else { + state.checkStateSize(ctxId, ctx.getMaxStateSize()); + stateSizeOk = state.isSizeOk(); + if (stateSizeOk) { + callback.onSuccess(); // State was updated but no calculation performed; + } + } + if (stateSizeOk) { + cfStateService.persistState(ctxId, state, callback); + } else { + removeStateAndRaiseSizeException(ctxId, CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(), callback); + } + } + + private void removeStateAndRaiseSizeException(CalculatedFieldEntityCtxId ctxId, CalculatedFieldException ex, TbCallback callback) { + // We remove the state, but remember that it is over-sized in a local map. + cfStateService.removeState(ctxId, new TbCallback() { + @Override + public void onSuccess() { + callback.onFailure(ex); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(ex); + } + }); + } + + private Map mapToArguments(CalculatedFieldCtx ctx, List data) { + return mapToArguments(ctx.getMainEntityArguments(), data); + } + + private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List data) { + var argNames = ctx.getLinkedEntityArguments().get(entityId); + if (argNames.isEmpty()) { + return Collections.emptyMap(); + } + return mapToArguments(argNames, data); + } + + private Map mapToArguments(Map argNames, List data) { + if (argNames.isEmpty()) { + return Collections.emptyMap(); + } + Map arguments = new HashMap<>(); + for (TsKvProto item : data) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); + String argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } + key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_ROLLING, null); + argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } + } + return arguments; + } + + private Map mapToArguments(CalculatedFieldCtx ctx, AttributeScopeProto scope, List attrDataList) { + return mapToArguments(ctx.getMainEntityArguments(), scope, attrDataList); + } + + private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List attrDataList) { + var argNames = ctx.getLinkedEntityArguments().get(entityId); + if (argNames.isEmpty()) { + return Collections.emptyMap(); + } + return mapToArguments(argNames, scope, attrDataList); + } + + private Map mapToArguments(Map argNames, AttributeScopeProto scope, List attrDataList) { + Map arguments = new HashMap<>(); + for (AttributeValueProto item : attrDataList) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); + String argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } + } + return arguments; + } + + private Map mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List removedAttrKeys) { + var argNames = ctx.getLinkedEntityArguments().get(entityId); + if (argNames.isEmpty()) { + return Collections.emptyMap(); + } + return mapToArgumentsWithDefaultValue(argNames, ctx.getArguments(), scope, removedAttrKeys); + } + + private Map mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, AttributeScopeProto scope, List removedAttrKeys) { + return mapToArgumentsWithDefaultValue(ctx.getMainEntityArguments(), ctx.getArguments(), scope, removedAttrKeys); + } + + private Map mapToArgumentsWithDefaultValue(Map argNames, Map configArguments, AttributeScopeProto scope, List removedAttrKeys) { + Map arguments = new HashMap<>(); + for (String removedKey : removedAttrKeys) { + ReferencedEntityKey key = new ReferencedEntityKey(removedKey, ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); + String argName = argNames.get(key); + if (argName != null) { + Argument argument = configArguments.get(argName); + String defaultValue = (argument != null) ? argument.getDefaultValue() : null; + arguments.put(argName, StringUtils.isNotEmpty(defaultValue) + ? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) + : new SingleValueArgumentEntry()); + + } + } + return arguments; + } + + private Map mapToArgumentsWithFetchedValue(CalculatedFieldCtx ctx, List removedTelemetryKeys) { + Map deletedArguments = ctx.getArguments().entrySet().stream() + .filter(entry -> removedTelemetryKeys.contains(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, entityId, deletedArguments); + + fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); + return fetchedArgs; + } + + private static List getCalculatedFieldIds(CalculatedFieldTelemetryMsgProto proto) { + List cfIds = new LinkedList<>(); + for (var cfId : proto.getPreviousCalculatedFieldsList()) { + cfIds.add(new CalculatedFieldId(new UUID(cfId.getCalculatedFieldIdMSB(), cfId.getCalculatedFieldIdLSB()))); + } + return cfIds; + } + + private UUID toTbMsgId(CalculatedFieldTelemetryMsgProto proto) { + if (proto.getTbMsgIdMSB() != 0 && proto.getTbMsgIdLSB() != 0) { + return new UUID(proto.getTbMsgIdMSB(), proto.getTbMsgIdLSB()); + } + return null; + } + + private TbMsgType toTbMsgType(CalculatedFieldTelemetryMsgProto proto) { + if (!proto.getTbMsgType().isEmpty()) { + return TbMsgType.valueOf(proto.getTbMsgType()); + } + return null; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldException.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldException.java new file mode 100644 index 0000000000..70c8dfbfd2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldException.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Builder; +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.Map; +import java.util.UUID; + +@Getter +@Builder +public class CalculatedFieldException extends Exception { + + private final CalculatedFieldCtx ctx; + private final EntityId eventEntity; + private final UUID msgId; + private final TbMsgType msgType; + private Map arguments; + private String errorMessage; + private Exception cause; + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldLinkedTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldLinkedTelemetryMsg.java new file mode 100644 index 0000000000..3e0fba2627 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldLinkedTelemetryMsg.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; + +@Data +public class CalculatedFieldLinkedTelemetryMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldLinkedTelemetryMsgProto proto; + private final TbCallback callback; + + + @Override + public MsgType getMsgType() { + return MsgType.CF_LINKED_TELEMETRY_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java new file mode 100644 index 0000000000..a5c935e83f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActorCtx; +import org.thingsboard.server.actors.TbActorException; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; + +/** + * Created by ashvayka on 15.03.18. + */ +@Slf4j +public class CalculatedFieldManagerActor extends AbstractCalculatedFieldActor { + + private final CalculatedFieldManagerMessageProcessor processor; + + public CalculatedFieldManagerActor(ActorSystemContext systemContext, TenantId tenantId) { + super(systemContext, tenantId); + this.processor = new CalculatedFieldManagerMessageProcessor(systemContext, tenantId); + } + + @Override + public void init(TbActorCtx ctx) throws TbActorException { + super.init(ctx); + log.debug("[{}] Starting CF manager actor.", processor.tenantId); + try { + processor.init(ctx); + log.debug("[{}] CF manager actor started.", processor.tenantId); + } catch (Exception e) { + log.warn("[{}] Unknown failure", processor.tenantId, e); + throw new TbActorException("Failed to initialize manager actor", e); + } + } + + @Override + protected boolean doProcessCfMsg(ToCalculatedFieldSystemMsg msg) throws CalculatedFieldException { + switch (msg.getMsgType()) { + case CF_PARTITIONS_CHANGE_MSG: + processor.onPartitionChange((CalculatedFieldPartitionChangeMsg) msg); + break; + case CF_INIT_MSG: + processor.onFieldInitMsg((CalculatedFieldInitMsg) msg); + break; + case CF_LINK_INIT_MSG: + processor.onLinkInitMsg((CalculatedFieldLinkInitMsg) msg); + break; + case CF_STATE_RESTORE_MSG: + processor.onStateRestoreMsg((CalculatedFieldStateRestoreMsg) msg); + break; + case CF_ENTITY_LIFECYCLE_MSG: + processor.onEntityLifecycleMsg((CalculatedFieldEntityLifecycleMsg) msg); + break; + case CF_TELEMETRY_MSG: + processor.onTelemetryMsg((CalculatedFieldTelemetryMsg) msg); + break; + case CF_LINKED_TELEMETRY_MSG: + processor.onLinkedTelemetryMsg((CalculatedFieldLinkedTelemetryMsg) msg); + break; + default: + return false; + } + return true; + } + + @Override + void logProcessingException(Exception e) { + log.warn("[{}] Processing failure", tenantId, e); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActorCreator.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActorCreator.java new file mode 100644 index 0000000000..99bf3cdbe9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActorCreator.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActor; +import org.thingsboard.server.actors.TbActorId; +import org.thingsboard.server.actors.TbEntityActorId; +import org.thingsboard.server.actors.TbStringActorId; +import org.thingsboard.server.actors.service.ContextBasedCreator; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +public class CalculatedFieldManagerActorCreator extends ContextBasedCreator { + + private final TenantId tenantId; + + public CalculatedFieldManagerActorCreator(ActorSystemContext context, TenantId tenantId) { + super(context); + this.tenantId = tenantId; + } + + @Override + public TbActorId createActorId() { + return new TbStringActorId("CFM|" + tenantId); + } + + @Override + public TbActor createActor() { + return new CalculatedFieldManagerActor(context, tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java new file mode 100644 index 0000000000..d26d89626e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -0,0 +1,471 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.DebugModeUtil; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActorCtx; +import org.thingsboard.server.actors.TbActorRef; +import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; +import org.thingsboard.server.actors.service.DefaultActorService; +import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CopyOnWriteArrayList; + +import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; + + +/** + * @author Andrew Shvayka + */ +@Slf4j +public class CalculatedFieldManagerMessageProcessor extends AbstractContextAwareMsgProcessor { + + private final Map calculatedFields = new HashMap<>(); + private final Map> entityIdCalculatedFields = new HashMap<>(); + private final Map> entityIdCalculatedFieldLinks = new HashMap<>(); + + private final CalculatedFieldProcessingService cfExecService; + private final CalculatedFieldStateService cfStateService; + private final CalculatedFieldEntityProfileCache cfEntityCache; + private final CalculatedFieldService cfDaoService; + private final TbAssetProfileCache assetProfileCache; + private final TbDeviceProfileCache deviceProfileCache; + protected final TenantId tenantId; + + protected TbActorCtx ctx; + + CalculatedFieldManagerMessageProcessor(ActorSystemContext systemContext, TenantId tenantId) { + super(systemContext); + this.cfEntityCache = systemContext.getCalculatedFieldEntityProfileCache(); + this.cfExecService = systemContext.getCalculatedFieldProcessingService(); + this.cfStateService = systemContext.getCalculatedFieldStateService(); + this.cfDaoService = systemContext.getCalculatedFieldService(); + this.assetProfileCache = systemContext.getAssetProfileCache(); + this.deviceProfileCache = systemContext.getDeviceProfileCache(); + this.tenantId = tenantId; + } + + void init(TbActorCtx ctx) { + this.ctx = ctx; + } + + public void onFieldInitMsg(CalculatedFieldInitMsg msg) throws CalculatedFieldException { + log.info("[{}] Processing CF init message.", msg.getCf().getId()); + var cf = msg.getCf(); + var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); + try { + cfCtx.init(); + } catch (Exception e) { + throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); + } + calculatedFields.put(cf.getId(), cfCtx); + // We use copy on write lists to safely pass the reference to another actor for the iteration. + // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); + msg.getCallback().onSuccess(); + } + + public void onLinkInitMsg(CalculatedFieldLinkInitMsg msg) { + log.info("[{}] Processing CF link init message for entity [{}].", msg.getLink().getCalculatedFieldId(), msg.getLink().getEntityId()); + var link = msg.getLink(); + // We use copy on write lists to safely pass the reference to another actor for the iteration. + // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) + entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link); + msg.getCallback().onSuccess(); + } + + public void onStateRestoreMsg(CalculatedFieldStateRestoreMsg msg) { + var cfId = msg.getId().cfId(); + var calculatedField = calculatedFields.get(cfId); + + if (calculatedField != null) { + msg.getState().setRequiredArguments(calculatedField.getArgNames()); + log.info("Pushing CF state restore msg to specific actor [{}]", msg.getId().entityId()); + getOrCreateActor(msg.getId().entityId()).tell(msg); + } else { + cfStateService.removeState(msg.getId(), msg.getCallback()); + } + } + + public void onEntityLifecycleMsg(CalculatedFieldEntityLifecycleMsg msg) throws CalculatedFieldException { + log.info("Processing entity lifecycle event: [{}] for entity: [{}]", msg.getData().getEvent(), msg.getData().getEntityId()); + var entityType = msg.getData().getEntityId().getEntityType(); + var event = msg.getData().getEvent(); + switch (entityType) { + case CALCULATED_FIELD: { + switch (event) { + case CREATED: + onCfCreated(msg.getData(), msg.getCallback()); + break; + case UPDATED: + onCfUpdated(msg.getData(), msg.getCallback()); + break; + case DELETED: + onCfDeleted(msg.getData(), msg.getCallback()); + break; + default: + msg.getCallback().onSuccess(); + break; + } + break; + } + case DEVICE: + case ASSET: { + switch (event) { + case CREATED: + onEntityCreated(msg.getData(), msg.getCallback()); + break; + case UPDATED: + onEntityUpdated(msg.getData(), msg.getCallback()); + break; + case DELETED: + onEntityDeleted(msg.getData(), msg.getCallback()); + break; + default: + msg.getCallback().onSuccess(); + break; + } + break; + } + default: { + msg.getCallback().onSuccess(); + } + } + } + + private void onEntityCreated(ComponentLifecycleMsg msg, TbCallback callback) { + EntityId entityId = msg.getEntityId(); + EntityId profileId = getProfileId(tenantId, entityId); + cfEntityCache.add(tenantId, profileId, entityId); + var entityIdFields = getCalculatedFieldsByEntityId(entityId); + var profileIdFields = getCalculatedFieldsByEntityId(profileId); + var fieldsCount = entityIdFields.size() + profileIdFields.size(); + if (fieldsCount > 0) { + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); + entityIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); + profileIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); + } else { + callback.onSuccess(); + } + } + + private void onEntityUpdated(ComponentLifecycleMsg msg, TbCallback callback) { + if (msg.getOldProfileId() != null && msg.getOldProfileId() != msg.getProfileId()) { + cfEntityCache.update(tenantId, msg.getOldProfileId(), msg.getProfileId(), msg.getEntityId()); + var oldProfileCfs = getCalculatedFieldsByEntityId(msg.getOldProfileId()); + var newProfileCfs = getCalculatedFieldsByEntityId(msg.getProfileId()); + var fieldsCount = oldProfileCfs.size() + newProfileCfs.size(); + if (fieldsCount > 0) { + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); + var entityId = msg.getEntityId(); + oldProfileCfs.forEach(ctx -> deleteCfForEntity(entityId, ctx.getCfId(), callback)); + newProfileCfs.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); + } else { + callback.onSuccess(); + } + } + } + + private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { + cfEntityCache.evict(tenantId, msg.getEntityId()); + log.info("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); + getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); + } + + private void onCfCreated(ComponentLifecycleMsg msg, TbCallback callback) throws CalculatedFieldException { + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + if (calculatedFields.containsKey(cfId)) { + log.warn("[{}] CF was already initialized [{}]", tenantId, cfId); + callback.onSuccess(); + } else { + var cf = cfDaoService.findById(msg.getTenantId(), cfId); + if (cf == null) { + log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); + callback.onSuccess(); + } else { + var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); + try { + cfCtx.init(); + } catch (Exception e) { + throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); + } + calculatedFields.put(cf.getId(), cfCtx); + // We use copy on write lists to safely pass the reference to another actor for the iteration. + // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); + addLinks(cf); + initCf(cfCtx, callback, false); + } + } + } + + private void onCfUpdated(ComponentLifecycleMsg msg, TbCallback callback) throws CalculatedFieldException { + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + var oldCfCtx = calculatedFields.get(cfId); + if (oldCfCtx == null) { + onCfCreated(msg, callback); + } else { + var newCf = cfDaoService.findById(msg.getTenantId(), cfId); + if (newCf == null) { + log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); + callback.onSuccess(); + } else { + var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); + try { + newCfCtx.init(); + } catch (Exception e) { + throw CalculatedFieldException.builder().ctx(newCfCtx).eventEntity(newCfCtx.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); + } + calculatedFields.put(newCf.getId(), newCfCtx); + List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); + List newCfList = new CopyOnWriteArrayList<>(); + boolean found = false; + for (CalculatedFieldCtx oldCtx : oldCfList) { + if (oldCtx.getCfId().equals(newCf.getId())) { + newCfList.add(newCfCtx); + found = true; + } else { + newCfList.add(oldCtx); + } + } + if (!found) { + newCfList.add(newCfCtx); + } + entityIdCalculatedFields.put(newCf.getEntityId(), newCfList); + + deleteLinks(oldCfCtx); + addLinks(newCf); + + // We use copy on write lists to safely pass the reference to another actor for the iteration. + // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) + var stateChanges = newCfCtx.hasStateChanges(oldCfCtx); + if (stateChanges || newCfCtx.hasOtherSignificantChanges(oldCfCtx)) { + initCf(newCfCtx, callback, stateChanges); + } else { + callback.onSuccess(); + } + } + } + } + + private void onCfDeleted(ComponentLifecycleMsg msg, TbCallback callback) { + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + var cfCtx = calculatedFields.remove(cfId); + if (cfCtx == null) { + log.warn("[{}] CF was already deleted [{}]", tenantId, cfId); + callback.onSuccess(); + } else { + entityIdCalculatedFields.get(cfCtx.getEntityId()).remove(cfCtx); + deleteLinks(cfCtx); + + EntityId entityId = cfCtx.getEntityId(); + EntityType entityType = cfCtx.getEntityId().getEntityType(); + if (isProfileEntity(entityType)) { + var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, entityId); + if (!entityIds.isEmpty()) { + //TODO: no need to do this if we cache all created actors and know which one belong to us; + var multiCallback = new MultipleTbCallback(entityIds.size(), callback); + entityIds.forEach(id -> deleteCfForEntity(entityId, cfId, multiCallback)); + } else { + callback.onSuccess(); + } + } else { + deleteCfForEntity(entityId, cfId, callback); + } + } + } + + public void onTelemetryMsg(CalculatedFieldTelemetryMsg msg) { + EntityId entityId = msg.getEntityId(); + log.info("Received telemetry msg from entity [{}]", entityId); + // 2 = 1 for CF processing + 1 for links processing + MultipleTbCallback callback = new MultipleTbCallback(2, msg.getCallback()); + // process all cfs related to entity, or it's profile; + var entityIdFields = getCalculatedFieldsByEntityId(entityId); + var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); + if (!entityIdFields.isEmpty() || !profileIdFields.isEmpty()) { + log.info("Pushing telemetry msg to specific actor [{}]", entityId); + getOrCreateActor(entityId).tell(new EntityCalculatedFieldTelemetryMsg(msg, entityIdFields, profileIdFields, callback)); + } else { + callback.onSuccess(); + } + // process all links (if any); + List linkedCalculatedFields = filterCalculatedFieldLinks(msg); + var linksSize = linkedCalculatedFields.size(); + if (linksSize > 0) { + cfExecService.pushMsgToLinks(msg, linkedCalculatedFields, callback); + } else { + callback.onSuccess(); + } + } + + public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsg msg) { + EntityId sourceEntityId = msg.getEntityId(); + log.info("Received linked telemetry msg from entity [{}]", sourceEntityId); + var proto = msg.getProto(); + var linksList = proto.getLinksList(); + for (var linkProto : linksList) { + var link = fromProto(linkProto); + var targetEntityId = link.entityId(); + var targetEntityType = targetEntityId.getEntityType(); + var cf = calculatedFields.get(link.cfId()); + if (EntityType.DEVICE_PROFILE.equals(targetEntityType) || EntityType.ASSET_PROFILE.equals(targetEntityType)) { + // iterate over all entities that belong to profile and push the message for corresponding CF + var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, targetEntityId); + if (!entityIds.isEmpty()) { + MultipleTbCallback callback = new MultipleTbCallback(entityIds.size(), msg.getCallback()); + var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, callback); + entityIds.forEach(entityId -> { + log.info("Pushing linked telemetry msg to specific actor [{}]", entityId); + getOrCreateActor(entityId).tell(newMsg); + }); + } else { + msg.getCallback().onSuccess(); + } + } else { + log.info("Pushing linked telemetry msg to specific actor [{}]", targetEntityId); + var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, msg.getCallback()); + getOrCreateActor(targetEntityId).tell(newMsg); + } + } + } + + private List filterCalculatedFieldLinks(CalculatedFieldTelemetryMsg msg) { + EntityId entityId = msg.getEntityId(); + var proto = msg.getProto(); + List result = new ArrayList<>(); + for (var link : getCalculatedFieldLinksByEntityId(entityId)) { + CalculatedFieldCtx ctx = calculatedFields.get(link.getCalculatedFieldId()); + if (ctx.linkMatches(entityId, proto)) { + result.add(ctx.toCalculatedFieldEntityCtxId()); + } + } + return result; + } + + private List getCalculatedFieldsByEntityId(EntityId entityId) { + if (entityId == null) { + return Collections.emptyList(); + } + var result = entityIdCalculatedFields.get(entityId); + if (result == null) { + result = Collections.emptyList(); + } + return result; + } + + private List getCalculatedFieldLinksByEntityId(EntityId entityId) { + if (entityId == null) { + return Collections.emptyList(); + } + var result = entityIdCalculatedFieldLinks.get(entityId); + if (result == null) { + result = Collections.emptyList(); + } + return result; + } + + private void initCf(CalculatedFieldCtx cfCtx, TbCallback callback, boolean forceStateReinit) { + EntityId entityId = cfCtx.getEntityId(); + EntityType entityType = cfCtx.getEntityId().getEntityType(); + if (isProfileEntity(entityType)) { + var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, entityId); + if (!entityIds.isEmpty()) { + var multiCallback = new MultipleTbCallback(entityIds.size(), callback); + entityIds.forEach(id -> initCfForEntity(id, cfCtx, forceStateReinit, multiCallback)); + } else { + callback.onSuccess(); + } + } else { + initCfForEntity(entityId, cfCtx, forceStateReinit, callback); + } + } + + private void deleteCfForEntity(EntityId entityId, CalculatedFieldId cfId, TbCallback callback) { + log.info("Pushing delete CF msg to specific actor [{}]", entityId); + getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, cfId, callback)); + } + + private void initCfForEntity(EntityId entityId, CalculatedFieldCtx cfCtx, boolean forceStateReinit, TbCallback callback) { + log.info("Pushing entity init CF msg to specific actor [{}]", entityId); + getOrCreateActor(entityId).tell(new EntityInitCalculatedFieldMsg(tenantId, cfCtx, callback, forceStateReinit)); + } + + private static boolean isProfileEntity(EntityType entityType) { + return EntityType.DEVICE_PROFILE.equals(entityType) || EntityType.ASSET_PROFILE.equals(entityType); + } + + private EntityId getProfileId(TenantId tenantId, EntityId entityId) { + return switch (entityId.getEntityType()) { + case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); + default -> null; + }; + } + + private TbActorRef getOrCreateActor(EntityId entityId) { + return ctx.getOrCreateChildActor(new TbCalculatedFieldEntityActorId(entityId), + () -> DefaultActorService.CF_ENTITY_DISPATCHER_NAME, + () -> new CalculatedFieldEntityActorCreator(systemContext, tenantId, entityId), + () -> true); + } + + private void addLinks(CalculatedField newCf) { + var newLinks = newCf.getConfiguration().buildCalculatedFieldLinks(tenantId, newCf.getEntityId(), newCf.getId()); + newLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link)); + } + + private void deleteLinks(CalculatedFieldCtx cfCtx) { + var oldCf = cfCtx.getCalculatedField(); + var oldLinks = oldCf.getConfiguration().buildCalculatedFieldLinks(tenantId, oldCf.getEntityId(), oldCf.getId()); + oldLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).remove(link)); + } + + public void onPartitionChange(CalculatedFieldPartitionChangeMsg msg) { + ctx.broadcastToChildren(msg, true); + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java new file mode 100644 index 0000000000..19be7c02fa --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; + +@Data +public class CalculatedFieldStateRestoreMsg implements ToCalculatedFieldSystemMsg { + + private final CalculatedFieldEntityCtxId id; + private final CalculatedFieldState state; + + @Override + public MsgType getMsgType() { + return MsgType.CF_STATE_RESTORE_MSG; + } + + @Override + public TenantId getTenantId() { + return id.tenantId(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldTelemetryMsg.java new file mode 100644 index 0000000000..68cd149cdf --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldTelemetryMsg.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; + +@Data +public class CalculatedFieldTelemetryMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldTelemetryMsgProto proto; + private final TbCallback callback; + + + @Override + public MsgType getMsgType() { + return MsgType.CF_TELEMETRY_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldLinkedTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldLinkedTelemetryMsg.java new file mode 100644 index 0000000000..b83aeae416 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldLinkedTelemetryMsg.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.List; + +@Data +public class EntityCalculatedFieldLinkedTelemetryMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldTelemetryMsgProto proto; + private final CalculatedFieldCtx ctx; + private final TbCallback callback; + + @Override + public MsgType getMsgType() { + return MsgType.CF_LINKED_TELEMETRY_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldTelemetryMsg.java new file mode 100644 index 0000000000..8ded4b6028 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldTelemetryMsg.java @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.List; + +@Data +public class EntityCalculatedFieldTelemetryMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldTelemetryMsgProto proto; + // Both lists are effectively immutable in CalculatedFieldManagerMessageProcessor and must stay so. + private final List entityIdFields; + private final List profileIdFields; + private final TbCallback callback; + + public EntityCalculatedFieldTelemetryMsg(CalculatedFieldTelemetryMsg msg, + List entityIdFields, + List profileIdFields, + TbCallback callback) { + this.tenantId = msg.getTenantId(); + this.entityId = msg.getEntityId(); + this.proto = msg.getProto(); + this.entityIdFields = entityIdFields; + this.profileIdFields = profileIdFields; + this.callback = callback; + } + + @Override + public MsgType getMsgType() { + return MsgType.CF_ENTITY_TELEMETRY_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java new file mode 100644 index 0000000000..1e8990ff8d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.List; + +@Data +public class EntityInitCalculatedFieldMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final CalculatedFieldCtx ctx; + private final TbCallback callback; + private final boolean forceReinit; + + @Override + public MsgType getMsgType() { + return MsgType.CF_ENTITY_INIT_CF_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java new file mode 100644 index 0000000000..d1f4c9092e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TbCallback; + +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +public class MultipleTbCallback implements TbCallback { + @Getter + private final UUID id; + private final AtomicInteger counter; + private final TbCallback callback; + + public MultipleTbCallback(int count, TbCallback callback) { + id = UUID.randomUUID(); + this.counter = new AtomicInteger(count); + this.callback = callback; + } + + @Override + public void onSuccess() { + onSuccess(1); + } + + public void onSuccess(int number) { + log.trace("[{}][{}] onSuccess({})", id, callback.getId(), number); + if (counter.addAndGet(-number) <= 0) { + log.trace("[{}][{}] Done.", id, callback.getId()); + callback.onSuccess(); + } + } + + @Override + public void onFailure(Throwable t) { + log.warn("[{}][{}] onFailure.", id, callback.getId()); + callback.onFailure(t); + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 8bd3bcc81d..d162b6a09a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import io.netty.channel.EventLoopGroup; import lombok.extern.slf4j.Slf4j; import org.bouncycastle.util.Arrays; +import org.thingsboard.common.util.DebugModeUtil; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.api.MailService; @@ -64,7 +65,6 @@ import org.thingsboard.server.common.data.msg.TbNodeConnectionType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rule.RuleNode; -import org.thingsboard.common.util.DebugModeUtil; import org.thingsboard.server.common.data.rule.RuleNodeState; import org.thingsboard.server.common.data.script.ScriptLanguage; import org.thingsboard.server.common.msg.TbActorMsg; @@ -79,6 +79,7 @@ import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceCredentialsService; @@ -178,7 +179,7 @@ public class DefaultTbContext implements TbContext { .resetRuleNodeId() .build(); tbMsg.pushToStack(nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId()); - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getQueueName(), getTenantId(), tbMsg.getOriginator()); + TopicPartitionInfo tpi = resolvePartition(msg); doEnqueue(tpi, tbMsg, new SimpleTbQueueCallback(md -> ack(msg), t -> tellFailure(msg, t))); } @@ -195,8 +196,7 @@ public class DefaultTbContext implements TbContext { @Override public void enqueue(TbMsg tbMsg, Runnable onSuccess, Consumer onFailure) { - TopicPartitionInfo tpi = mainCtx.resolve(ServiceType.TB_RULE_ENGINE, getQueueName(), getTenantId(), tbMsg.getOriginator()); - enqueue(tpi, tbMsg, onFailure, onSuccess); + enqueue(tbMsg, tbMsg.getQueueName(), onSuccess, onFailure); } @Override @@ -897,6 +897,11 @@ public class DefaultTbContext implements TbContext { return mainCtx.getSlackService(); } + @Override + public CalculatedFieldService getCalculatedFieldService() { + return mainCtx.getCalculatedFieldService(); + } + @Override public boolean isExternalNodeForceAck() { return mainCtx.isExternalNodeForceAck(); diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index 6c8f253138..c2131d9e74 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -49,6 +49,8 @@ public class DefaultActorService extends TbApplicationEventListener deletedDevices; + private TbActorRef cfActor; private TenantActor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext, tenantId); @@ -95,6 +98,11 @@ public class TenantActor extends RuleChainManagerActor { } else { log.info("[{}] Skip init of the rule chains due to API limits", tenantId); } + //TODO: IM - extend API usage to have CF Exec Enabled? Not in 4.0; + cfActor = ctx.getOrCreateChildActor(new TbStringActorId("CFM|" + tenantId), + () -> DefaultActorService.CF_MANAGER_DISPATCHER_NAME, + () -> new CalculatedFieldManagerActorCreator(systemContext, tenantId), + () -> true); } catch (Exception e) { log.info("Failed to check ApiUsage \"ReExecEnabled\"!!!", e); cantFindTenant = true; @@ -159,12 +167,31 @@ public class TenantActor extends RuleChainManagerActor { case RULE_CHAIN_TO_RULE_CHAIN_MSG: onRuleChainMsg((RuleChainAwareMsg) msg); break; + case CF_INIT_MSG: + case CF_LINK_INIT_MSG: + case CF_STATE_RESTORE_MSG: + case CF_PARTITIONS_CHANGE_MSG: + case CF_ENTITY_LIFECYCLE_MSG: + onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, true); + break; + case CF_TELEMETRY_MSG: + case CF_LINKED_TELEMETRY_MSG: + onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, false); + break; default: return false; } return true; } + private void onToCalculatedFieldSystemActorMsg(ToCalculatedFieldSystemMsg msg, boolean priority) { + if (priority) { + cfActor.tellWithHighPriority(msg); + } else { + cfActor.tell(msg); + } + } + private boolean isMyPartition(EntityId entityId) { return systemContext.resolve(ServiceType.TB_CORE, tenantId, entityId).isMyPartition(); } diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index cf9d6444a2..73e278389a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -70,6 +70,7 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.domain.Domain; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeInfo; @@ -80,6 +81,7 @@ import org.thingsboard.server.common.data.id.AlarmCommentId; import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceId; @@ -132,6 +134,7 @@ import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.ClaimDevicesService; @@ -367,6 +370,9 @@ public abstract class BaseController { @Autowired protected NotificationTargetService notificationTargetService; + @Autowired + protected CalculatedFieldService calculatedFieldService; + @Value("${server.log_controller_error_stack_trace}") @Getter private boolean logControllerErrorStackTrace; @@ -672,6 +678,9 @@ public abstract class BaseController { case MOBILE_APP_BUNDLE: checkMobileAppBundleId(new MobileAppBundleId(entityId.getId()), operation); return; + case CALCULATED_FIELD: + checkCalculatedFieldId(new CalculatedFieldId(entityId.getId()), operation); + return; default: checkEntityId(entityId, entitiesService::findEntityByTenantIdAndId, operation); } @@ -955,6 +964,10 @@ public abstract class BaseController { } } + protected CalculatedField checkCalculatedFieldId(CalculatedFieldId calculatedFieldId, Operation operation) throws ThingsboardException { + return checkEntityId(calculatedFieldId, calculatedFieldService::findById, operation); + } + protected HomeDashboardInfo getHomeDashboardInfo(SecurityUser securityUser, JsonNode additionalInfo) { HomeDashboardInfo homeDashboardInfo = extractHomeDashboardInfoFromAdditionalInfo(additionalInfo); if (homeDashboardInfo == null) { @@ -982,7 +995,8 @@ public abstract class BaseController { } return new HomeDashboardInfo(dashboardId, hideDashboardToolbar); } - } catch (Exception ignored) {} + } catch (Exception ignored) { + } return null; } diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java new file mode 100644 index 0000000000..fe85cf3f87 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -0,0 +1,269 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.event.EventType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.config.annotations.ApiOperation; +import org.thingsboard.server.dao.event.EventService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldScriptEngine; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldTbelScriptEngine; +import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +import static org.thingsboard.server.controller.ControllerConstants.CF_TEXT_SEARCH_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.ENTITY_ID_PARAM_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.ENTITY_TYPE_PARAM_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_END; +import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_START; +import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.SORT_PROPERTY_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH; +import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; +import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; + +@RestController +@TbCoreComponent +@RequestMapping("/api") +@RequiredArgsConstructor +@Slf4j +public class CalculatedFieldController extends BaseController { + + private final TbCalculatedFieldService tbCalculatedFieldService; + private final EventService eventService; + private final TbelInvokeService tbelInvokeService; + + public static final String CALCULATED_FIELD_ID = "calculatedFieldId"; + + public static final int TIMEOUT = 20; + + private static final String TEST_SCRIPT_EXPRESSION = "Execute the Script expression and return the result. The format of request: \n\n" + + MARKDOWN_CODE_BLOCK_START + + "{\n" + + " \"expression\": \"var temp = 0; foreach(element: temperature.values) {temp += element.value;} var avgTemperature = temp / temperature.values.size(); var adjustedTemperature = avgTemperature + 0.1 * humidity.value; return {\\\"adjustedTemperature\\\": adjustedTemperature};\",\n" + + " \"arguments\": {\n" + + " \"temperature\": {\n" + + " \"type\": \"TS_ROLLING\",\n" + + " \"timeWindow\": {\n" + + " \"startTs\": 1739775630002,\n" + + " \"endTs\": 65432211,\n" + + " \"limit\": 5\n" + + " },\n" + + " \"values\": [\n" + + " { \"ts\": 1739775639851, \"value\": 23 },\n" + + " { \"ts\": 1739775664561, \"value\": 43 },\n" + + " { \"ts\": 1739775713079, \"value\": 15 },\n" + + " { \"ts\": 1739775999522, \"value\": 34 },\n" + + " { \"ts\": 1739776228452, \"value\": 22 }\n" + + " ]\n" + + " },\n" + + " \"humidity\": { \"type\": \"SINGLE_VALUE\", \"ts\": 1739776478057, \"value\": 23 }\n" + + " }\n" + + "}" + + MARKDOWN_CODE_BLOCK_END + + "\n\n Expected result JSON contains \"output\" and \"error\"."; + + @ApiOperation(value = "Create Or Update Calculated Field (saveCalculatedField)", + notes = "Creates or Updates the Calculated Field. When creating calculated field, platform generates Calculated Field Id as " + UUID_WIKI_LINK + + "The newly created Calculated Field Id will be present in the response. " + + "Specify existing Calculated Field Id to update the calculated field. " + + "Referencing non-existing Calculated Field Id will cause 'Not Found' error. " + + "Remove 'id', 'tenantId' from the request body example (below) to create new Calculated Field entity. " + + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/calculatedField", method = RequestMethod.POST) + @ResponseBody + public CalculatedField saveCalculatedField(@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "A JSON value representing the calculated field.") + @RequestBody CalculatedField calculatedField) throws Exception { + calculatedField.setTenantId(getTenantId()); + checkEntity(calculatedField.getId(), calculatedField, Resource.CALCULATED_FIELD); + checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); + checkReferencedEntities(calculatedField.getConfiguration(), getCurrentUser()); + return tbCalculatedFieldService.save(calculatedField, getCurrentUser()); + } + + @ApiOperation(value = "Get Calculated Field (getCalculatedFieldById)", + notes = "Fetch the Calculated Field object based on the provided Calculated Field Id." + ) + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/calculatedField/{calculatedFieldId}", method = RequestMethod.GET) + @ResponseBody + public CalculatedField getCalculatedFieldById(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { + checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); + CalculatedField calculatedField = tbCalculatedFieldService.findById(calculatedFieldId, getCurrentUser()); + checkNotNull(calculatedField); + checkEntityId(calculatedField.getEntityId(), Operation.READ_CALCULATED_FIELD); + return calculatedField; + } + + @ApiOperation(value = "Get Calculated Fields by Entity Id (getCalculatedFieldsByEntityId)", + notes = "Fetch the Calculated Fields based on the provided Entity Id." + ) + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/{entityType}/{entityId}/calculatedFields", params = {"pageSize", "page"}, method = RequestMethod.GET) + @ResponseBody + public PageData getCalculatedFieldsByEntityId( + @Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true, schema = @Schema(defaultValue = "DEVICE")) @PathVariable("entityType") String entityType, + @Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @PathVariable("entityId") String entityIdStr, + @Parameter(description = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, + @Parameter(description = PAGE_NUMBER_DESCRIPTION, required = true) @RequestParam int page, + @Parameter(description = CF_TEXT_SEARCH_DESCRIPTION) @RequestParam(required = false) String textSearch, + @Parameter(description = SORT_PROPERTY_DESCRIPTION, schema = @Schema(allowableValues = {"createdTime", "name"})) @RequestParam(required = false) String sortProperty, + @Parameter(description = SORT_ORDER_DESCRIPTION, schema = @Schema(allowableValues = {"ASC", "DESC"})) @RequestParam(required = false) String sortOrder) throws ThingsboardException { + PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); + checkParameter("entityId", entityIdStr); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityIdStr); + checkEntityId(entityId, Operation.READ_CALCULATED_FIELD); + return checkNotNull(tbCalculatedFieldService.findAllByTenantIdAndEntityId(entityId, getCurrentUser(), pageLink)); + } + + @ApiOperation(value = "Delete Calculated Field (deleteCalculatedField)", + notes = "Deletes the calculated field. Referencing non-existing Calculated Field Id will cause an error." + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/calculatedField/{calculatedFieldId}", method = RequestMethod.DELETE) + @ResponseStatus(value = HttpStatus.OK) + public void deleteCalculatedField(@PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws Exception { + checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); + CalculatedField calculatedField = checkCalculatedFieldId(calculatedFieldId, Operation.DELETE); + checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); + tbCalculatedFieldService.delete(calculatedField, getCurrentUser()); + } + + @ApiOperation(value = "Get latest calculated field debug event (getLatestCalculatedFieldDebugEvent)", + notes = "Gets latest calculated field debug event for specified calculated field id. " + + "Referencing non-existing calculated field id will cause an error. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/calculatedField/{calculatedFieldId}/debug", method = RequestMethod.GET) + @ResponseBody + public JsonNode getLatestCalculatedFieldDebugEvent(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { + checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); + CalculatedField calculatedField = checkCalculatedFieldId(calculatedFieldId, Operation.READ); + checkEntityId(calculatedField.getEntityId(), Operation.READ_CALCULATED_FIELD); + TenantId tenantId = getCurrentUser().getTenantId(); + return Optional.ofNullable(eventService.findLatestEvents(tenantId, calculatedFieldId, EventType.DEBUG_CALCULATED_FIELD, 1)) + .flatMap(events -> events.stream().map(EventInfo::getBody).findFirst()) + .orElse(null); + } + + @ApiOperation(value = "Test Script expression", + notes = TEST_SCRIPT_EXPRESSION + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/calculatedField/testScript", method = RequestMethod.POST) + @ResponseBody + public JsonNode testScript( + @io.swagger.v3.oas.annotations.parameters.RequestBody(description = "Test calculated field TBEL expression.") + @RequestBody JsonNode inputParams) { + String expression = inputParams.get("expression").asText(); + Map arguments = Objects.requireNonNullElse( + JacksonUtil.convertValue(inputParams.get("arguments"), new TypeReference>() { + }), + Collections.emptyMap() + ); + ArrayList argNames = new ArrayList<>(arguments.keySet()); + + String output = ""; + String errorText = ""; + + try { + if (tbelInvokeService == null) { + throw new IllegalArgumentException("TBEL script engine is disabled!"); + } + + CalculatedFieldScriptEngine calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( + getTenantId(), + tbelInvokeService, + expression, + argNames.toArray(String[]::new) + ); + + Object[] args = argNames.stream() + .map(arguments::get) + .toArray(); + + JsonNode json = calculatedFieldScriptEngine.executeJsonAsync(args).get(TIMEOUT, TimeUnit.SECONDS); + output = JacksonUtil.toString(json); + } catch (Exception e) { + log.error("Error evaluating expression", e); + errorText = e.getMessage(); + } + + ObjectNode result = JacksonUtil.newObjectNode(); + result.put("output", output); + result.put("error", errorText); + return result; + } + + private & HasTenantId, I extends EntityId> void checkReferencedEntities(CalculatedFieldConfiguration calculatedFieldConfig, SecurityUser user) throws ThingsboardException { + List referencedEntityIds = calculatedFieldConfig.getReferencedEntities(); + for (EntityId referencedEntityId : referencedEntityIds) { + EntityType entityType = referencedEntityId.getEntityType(); + switch (entityType) { + case TENANT, CUSTOMER, ASSET, DEVICE -> checkEntityId(referencedEntityId, Operation.READ); + default -> throw new IllegalArgumentException("Calculated fields do not support '" + entityType + "' for referenced entities."); + } + } + + } + +} diff --git a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java index 4987e5c26f..e45d041950 100644 --- a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java +++ b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java @@ -96,6 +96,7 @@ public class ControllerConstants { protected static final String EDGE_TEXT_SEARCH_DESCRIPTION = "The case insensitive 'substring' filter based on the edge name."; protected static final String EVENT_TEXT_SEARCH_DESCRIPTION = "The value is not used in searching."; protected static final String AUDIT_LOG_TEXT_SEARCH_DESCRIPTION = "The case insensitive 'substring' filter based on one of the next properties: entityType, entityName, userName, actionType, actionStatus."; + protected static final String CF_TEXT_SEARCH_DESCRIPTION = "The case insensitive 'substring' filter based on the calculated field name."; protected static final String SORT_PROPERTY_DESCRIPTION = "Property of entity to sort by"; protected static final String SORT_ORDER_DESCRIPTION = "Sort order. ASC (ASCENDING) or DESC (DESCENDING)"; diff --git a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java index bc22313a03..4ee86871d6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java +++ b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java @@ -35,8 +35,8 @@ import org.thingsboard.server.common.data.SystemParams; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.mobile.qrCodeSettings.QrCodeSettings; import org.thingsboard.server.common.data.mobile.qrCodeSettings.QRCodeConfig; +import org.thingsboard.server.common.data.mobile.qrCodeSettings.QrCodeSettings; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.settings.UserSettings; import org.thingsboard.server.common.data.settings.UserSettingsType; @@ -46,6 +46,7 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; +import org.thingsboard.server.utils.DebugModeRateLimitsConfig; import java.util.Collections; import java.util.List; @@ -74,12 +75,6 @@ public class SystemInfoController extends BaseController { @Value("${debug.settings.default_duration:15}") private int defaultDebugDurationMinutes; - @Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.enabled:true}") - private boolean ruleChainDebugPerTenantLimitsEnabled; - - @Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") - private String ruleChainDebugPerTenantLimitsConfiguration; - @Autowired(required = false) private BuildProperties buildProperties; @@ -89,6 +84,9 @@ public class SystemInfoController extends BaseController { @Autowired private QrCodeSettingService qrCodeSettingService; + @Autowired + private DebugModeRateLimitsConfig debugModeRateLimitsConfig; + @PostConstruct public void init() { JsonNode info = buildInfoObject(); @@ -152,9 +150,14 @@ public class SystemInfoController extends BaseController { DefaultTenantProfileConfiguration tenantProfileConfiguration = tenantProfileCache.get(tenantId).getDefaultProfileConfiguration(); systemParams.setMaxResourceSize(tenantProfileConfiguration.getMaxResourceSize()); systemParams.setMaxDebugModeDurationMinutes(DebugModeUtil.getMaxDebugAllDuration(tenantProfileConfiguration.getMaxDebugModeDurationMinutes(), defaultDebugDurationMinutes)); - if (ruleChainDebugPerTenantLimitsEnabled) { - systemParams.setRuleChainDebugPerTenantLimitsConfiguration(ruleChainDebugPerTenantLimitsConfiguration); + if (debugModeRateLimitsConfig.isRuleChainDebugPerTenantLimitsEnabled()) { + systemParams.setRuleChainDebugPerTenantLimitsConfiguration(debugModeRateLimitsConfig.getRuleChainDebugPerTenantLimitsConfiguration()); + } + if (debugModeRateLimitsConfig.isCalculatedFieldDebugPerTenantLimitsEnabled()) { + systemParams.setCalculatedFieldDebugPerTenantLimitsConfiguration(debugModeRateLimitsConfig.getCalculatedFieldDebugPerTenantLimitsConfiguration()); } + systemParams.setMaxArgumentsPerCF(tenantProfileConfiguration.getMaxArgumentsPerCF()); + systemParams.setMaxDataPointsPerRollingArg(tenantProfileConfiguration.getMaxDataPointsPerRollingArg()); } systemParams.setMobileQrEnabled(Optional.ofNullable(qrCodeSettingService.findQrCodeSettings(TenantId.SYS_TENANT_ID)) .map(QrCodeSettings::getQrCodeConfig).map(QRCodeConfig::isShowOnHomePage) diff --git a/application/src/main/java/org/thingsboard/server/controller/TbResourceController.java b/application/src/main/java/org/thingsboard/server/controller/TbResourceController.java index 9c72b9a5ad..b23603f6a2 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TbResourceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TbResourceController.java @@ -41,6 +41,7 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.ResourceSubType; import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TbResourceDeleteResult; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.TbResourceInfoFilter; import org.thingsboard.server.common.data.exception.ThingsboardException; @@ -322,12 +323,14 @@ public class TbResourceController extends BaseController { notes = "Deletes the Resource. Referencing non-existing Resource Id will cause an error." + SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')") @DeleteMapping(value = "/resource/{resourceId}") - public void deleteResource(@Parameter(description = RESOURCE_ID_PARAM_DESCRIPTION) - @PathVariable("resourceId") String strResourceId) throws ThingsboardException { + public ResponseEntity deleteResource(@Parameter(description = RESOURCE_ID_PARAM_DESCRIPTION) + @PathVariable("resourceId") String strResourceId, + @RequestParam(name = "force", required = false) boolean force) throws ThingsboardException { checkParameter(RESOURCE_ID, strResourceId); TbResourceId resourceId = new TbResourceId(toUUID(strResourceId)); TbResource tbResource = checkResourceId(resourceId, Operation.DELETE); - tbResourceService.delete(tbResource, getCurrentUser()); + TbResourceDeleteResult tbResourceDeleteResult = tbResourceService.delete(tbResource, force, getCurrentUser()); + return (tbResourceDeleteResult.isSuccess() ? ResponseEntity.ok() : ResponseEntity.badRequest()).body(tbResourceDeleteResult); } private ResponseEntity downloadResourceIfChanged(String strResourceId, String etag) throws ThingsboardException { diff --git a/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java b/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java new file mode 100644 index 0000000000..a30b7218a1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.exception; + +public class CalculatedFieldStateException extends RuntimeException { + + public CalculatedFieldStateException(String message) { + super(message); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/BaseApiUsageState.java b/application/src/main/java/org/thingsboard/server/service/apiusage/BaseApiUsageState.java index 78176bb4f7..aa1117572b 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/BaseApiUsageState.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/BaseApiUsageState.java @@ -40,7 +40,8 @@ public abstract class BaseApiUsageState { private final Map gaugesReportCycles = new HashMap<>(); @Getter - private final ApiUsageState apiUsageState; + @Setter + private ApiUsageState apiUsageState; @Getter private volatile long currentCycleTs; @Getter diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java index cfc5040329..b0ab4edc27 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -15,13 +15,11 @@ */ package org.thingsboard.server.service.apiusage; -import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.ListenableFuture; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; @@ -92,15 +90,7 @@ import java.util.stream.Collectors; public class DefaultTbApiUsageStateService extends AbstractPartitionBasedService implements TbApiUsageStateService { public static final String HOURLY = "Hourly"; - public static final FutureCallback VOID_CALLBACK = new FutureCallback() { - @Override - public void onSuccess(@Nullable Void result) { - } - @Override - public void onFailure(Throwable t) { - } - }; private final PartitionService partitionService; private final TenantService tenantService; private final TimeseriesService tsService; @@ -219,7 +209,6 @@ public class DefaultTbApiUsageStateService extends AbstractPartitionBasedService .tenantId(tenantId) .entityId(usageState.getApiUsageState().getId()) .entries(updatedEntries) - .callback(VOID_CALLBACK) .build()); if (!result.isEmpty()) { persistAndNotify(usageState, result); @@ -331,7 +320,6 @@ public class DefaultTbApiUsageStateService extends AbstractPartitionBasedService .tenantId(tenantId) .entityId(id) .entries(profileThresholds) - .callback(VOID_CALLBACK) .build()); } } @@ -355,7 +343,8 @@ public class DefaultTbApiUsageStateService extends AbstractPartitionBasedService private void persistAndNotify(BaseApiUsageState state, Map result) { log.info("[{}] Detected update of the API state for {}: {}", state.getEntityId(), state.getEntityType(), result); - apiUsageStateService.update(state.getApiUsageState()); + ApiUsageState updatedState = apiUsageStateService.update(state.getApiUsageState()); + state.setApiUsageState(updatedState); long ts = System.currentTimeMillis(); List stateTelemetry = new ArrayList<>(); result.forEach((apiFeature, aState) -> stateTelemetry.add(new BasicTsKvEntry(ts, new StringDataEntry(apiFeature.getApiStateKey(), aState.name())))); @@ -363,7 +352,6 @@ public class DefaultTbApiUsageStateService extends AbstractPartitionBasedService .tenantId(state.getTenantId()) .entityId(state.getApiUsageState().getId()) .entries(stateTelemetry) - .callback(VOID_CALLBACK) .build()); if (state.getEntityType() == EntityType.TENANT && !state.getEntityId().equals(TenantId.SYS_TENANT_ID)) { @@ -456,7 +444,6 @@ public class DefaultTbApiUsageStateService extends AbstractPartitionBasedService .tenantId(state.getTenantId()) .entityId(state.getApiUsageState().getId()) .entries(counts) - .callback(VOID_CALLBACK) .build()); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java new file mode 100644 index 0000000000..0163d65d98 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java @@ -0,0 +1,72 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.exception.CalculatedFieldStateException; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; + +import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; +import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; + +public abstract class AbstractCalculatedFieldStateService implements CalculatedFieldStateService { + + @Autowired + private ActorSystemContext actorSystemContext; + + protected PartitionedQueueConsumerManager> eventConsumer; + + @Override + public void init(PartitionedQueueConsumerManager> eventConsumer) { + this.eventConsumer = eventConsumer; + } + + @Override + public final void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + if (state.isSizeExceedsLimit()) { + throw new CalculatedFieldStateException("State size exceeds the maximum allowed limit. The state will not be persisted to RocksDB."); + } + doPersist(stateId, toProto(stateId, state), callback); + } + + protected abstract void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback); + + @Override + public final void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback) { + doRemove(stateId, callback); + } + + protected abstract void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback); + + protected void processRestoredState(CalculatedFieldStateProto stateMsg) { + var id = fromProto(stateMsg.getId()); + var state = fromProto(stateMsg); + processRestoredState(id, state); + } + + protected void processRestoredState(CalculatedFieldEntityCtxId id, CalculatedFieldState state) { + actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(id, state)); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java new file mode 100644 index 0000000000..fb63432fed --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.List; + +public interface CalculatedFieldCache { + + CalculatedField getCalculatedField(CalculatedFieldId calculatedFieldId); + + List getCalculatedFieldsByEntityId(EntityId entityId); + + List getCalculatedFieldLinksByEntityId(EntityId entityId); + + CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId); + + List getCalculatedFieldCtxsByEntityId(EntityId entityId); + + void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + void evict(CalculatedFieldId calculatedFieldId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldInitService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldInitService.java new file mode 100644 index 0000000000..6505dae581 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldInitService.java @@ -0,0 +1,19 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +public interface CalculatedFieldInitService { +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java new file mode 100644 index 0000000000..847caccaff --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; + +import java.util.List; +import java.util.Map; + +public interface CalculatedFieldProcessingService { + + ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); + + Map fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map arguments); + + void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculationResult, List cfIds, TbCallback callback); + + void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List linkedCalculatedFields, TbCallback callback); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java new file mode 100644 index 0000000000..b688993a8d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import com.google.common.util.concurrent.FutureCallback; +import org.thingsboard.rule.engine.api.AttributesDeleteRequest; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; +import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; + +import java.util.List; + +public interface CalculatedFieldQueueService { + + /** + * Filter CFs based on the request entity. Push to the queue if any matching CF exist; + * + * @param request - telemetry save request; + * @param callback + */ + void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback); + + void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback); + + void pushRequestToQueue(AttributesDeleteRequest request, List result, FutureCallback callback); + + void pushRequestToQueue(TimeseriesDeleteRequest request, List result, FutureCallback callback); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java new file mode 100644 index 0000000000..8eb27395c1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.configuration.OutputType; + +@Data +public final class CalculatedFieldResult { + + private final OutputType type; + private final AttributeScope scope; + private final JsonNode result; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java new file mode 100644 index 0000000000..109f13f183 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.exception.CalculatedFieldStateException; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; + +import java.util.Set; + +public interface CalculatedFieldStateService { + + void init(PartitionedQueueConsumerManager> eventConsumer); + + void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) throws CalculatedFieldStateException; + + void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback); + + void restore(Set partitions); + + void stop(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java b/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java new file mode 100644 index 0000000000..f95227bc24 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import org.rocksdb.Options; +import org.rocksdb.WriteOptions; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.edqs.util.TbRocksDb; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='in-memory'") +public class CfRocksDb extends TbRocksDb { + + public CfRocksDb(@Value("${queue.calculated_fields.rocks_db_path:${user.home}/.rocksdb/cf_states}") String path) { + super(path, new Options().setCreateIfMissing(true), new WriteOptions().setSync(true)); + } + + @PostConstruct + @Override + public void init() { + super.init(); + } + + @PreDestroy + @Override + public void close() { + super.close(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java new file mode 100644 index 0000000000..2282124bd5 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -0,0 +1,188 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultCalculatedFieldCache implements CalculatedFieldCache { + + private static final Integer UNKNOWN_PARTITION = -1; + + private final Lock calculatedFieldFetchLock = new ReentrantLock(); + + private final CalculatedFieldService calculatedFieldService; + private final TbelInvokeService tbelInvokeService; + private final ActorSystemContext actorSystemContext; + private final ApiLimitService apiLimitService; + + private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> entityIdCalculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); + + @Value("${calculatedField.initFetchPackSize:50000}") + @Getter + private int initFetchPackSize; + + @AfterStartUp(order = AfterStartUp.CF_READ_CF_SERVICE) + public void init() { + //TODO: move to separate place to avoid circular references with the ActorSystemContext (@Lazy for tsSubService) + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + cfs.forEach(cf -> { + calculatedFields.putIfAbsent(cf.getId(), cf); + actorSystemContext.tell(new CalculatedFieldInitMsg(cf.getTenantId(), cf)); + }); + calculatedFields.values().forEach(cf -> { + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cf); + }); + PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); + cfls.forEach(link -> { + calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new CopyOnWriteArrayList<>()).add(link); + actorSystemContext.tell(new CalculatedFieldLinkInitMsg(link.getTenantId(), link)); + }); + calculatedFieldLinks.values().stream() + .flatMap(List::stream) + .forEach(link -> + entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link) + ); + } + + @Override + public CalculatedField getCalculatedField(CalculatedFieldId calculatedFieldId) { + return calculatedFields.get(calculatedFieldId); + } + + @Override + public List getCalculatedFieldsByEntityId(EntityId entityId) { + return entityIdCalculatedFields.getOrDefault(entityId, new CopyOnWriteArrayList<>()); + } + + @Override + public List getCalculatedFieldLinksByEntityId(EntityId entityId) { + return entityIdCalculatedFieldLinks.getOrDefault(entityId, new CopyOnWriteArrayList<>()); + } + + @Override + public CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId) { + CalculatedFieldCtx ctx = calculatedFieldsCtx.get(calculatedFieldId); + if (ctx == null) { + calculatedFieldFetchLock.lock(); + try { + ctx = calculatedFieldsCtx.get(calculatedFieldId); + if (ctx == null) { + CalculatedField calculatedField = getCalculatedField(calculatedFieldId); + if (calculatedField != null) { + ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService, apiLimitService); + ctx.init(); + calculatedFieldsCtx.put(calculatedFieldId, ctx); + log.debug("[{}] Put calculated field ctx into cache: {}", calculatedFieldId, ctx); + } + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found calculated field ctx in cache: {}", calculatedFieldId, ctx); + return ctx; + } + + @Override + public List getCalculatedFieldCtxsByEntityId(EntityId entityId) { + if (entityId == null) { + return Collections.emptyList(); + } + return getCalculatedFieldsByEntityId(entityId).stream() + .map(cf -> getCalculatedFieldCtx(cf.getId())) + .toList(); + } + + @Override + public void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + calculatedFieldFetchLock.lock(); + try { + CalculatedField calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); + EntityId cfEntityId = calculatedField.getEntityId(); + + calculatedFields.put(calculatedFieldId, calculatedField); + + entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new CopyOnWriteArrayList<>()).add(calculatedField); + + CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); + calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); + + configuration.getReferencedEntities().stream() + .filter(referencedEntityId -> !referencedEntityId.equals(cfEntityId)) + .forEach(referencedEntityId -> { + entityIdCalculatedFieldLinks.computeIfAbsent(referencedEntityId, entityId -> new CopyOnWriteArrayList<>()) + .add(configuration.buildCalculatedFieldLink(tenantId, referencedEntityId, calculatedFieldId)); + }); + } finally { + calculatedFieldFetchLock.unlock(); + } + } + + @Override + public void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + evict(calculatedFieldId); + addCalculatedField(tenantId, calculatedFieldId); + } + + @Override + public void evict(CalculatedFieldId calculatedFieldId) { + CalculatedField oldCalculatedField = calculatedFields.remove(calculatedFieldId); + log.debug("[{}] evict calculated field from cache: {}", calculatedFieldId, oldCalculatedField); + calculatedFieldLinks.remove(calculatedFieldId); + log.debug("[{}] evict calculated field from cached calculated fields by entity id: {}", calculatedFieldId, oldCalculatedField); + entityIdCalculatedFields.forEach((entityId, calculatedFields) -> calculatedFields.removeIf(cf -> cf.getId().equals(calculatedFieldId))); + log.debug("[{}] evict calculated field links from cache: {}", calculatedFieldId, oldCalculatedField); + calculatedFieldsCtx.remove(calculatedFieldId); + log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); + entityIdCalculatedFieldLinks.forEach((entityId, calculatedFieldLinks) -> calculatedFieldLinks.removeIf(link -> link.getCalculatedFieldId().equals(calculatedFieldId))); + log.debug("[{}] evict calculated field links from cached links by entity id: {}", calculatedFieldId, oldCalculatedField); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java new file mode 100644 index 0000000000..9442329edb --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; + +@Slf4j +@Service +@TbRuleEngineComponent +@RequiredArgsConstructor +public class DefaultCalculatedFieldInitService implements CalculatedFieldInitService { + + private final CalculatedFieldEntityProfileCache entityProfileCache; + private final AssetService assetService; + private final DeviceService deviceService; + + @Value("${calculated_fields.init_fetch_pack_size:50000}") + @Getter + private int initFetchPackSize; + + @AfterStartUp(order = AfterStartUp.CF_READ_PROFILE_ENTITIES_SERVICE) + public void initCalculatedFieldDefinitions() { + PageDataIterable deviceIdInfos = new PageDataIterable<>(deviceService::findProfileEntityIdInfos, initFetchPackSize); + for (ProfileEntityIdInfo idInfo : deviceIdInfos) { + log.trace("Processing device record: {}", idInfo); + entityProfileCache.add(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId()); + } + PageDataIterable assetIdInfos = new PageDataIterable<>(assetService::findProfileEntityIdInfos, initFetchPackSize); + for (ProfileEntityIdInfo idInfo : assetIdInfos) { + log.trace("Processing asset record: {}", idInfo); + entityProfileCache.add(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId()); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java new file mode 100644 index 0000000000..f69f5cd50b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -0,0 +1,324 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.math.NumberUtils; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; +import org.thingsboard.server.actors.calculatedField.MultipleTbCallback; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +import static org.thingsboard.server.common.data.DataConstants.SCOPE; +import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; + +@TbRuleEngineComponent +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultCalculatedFieldProcessingService implements CalculatedFieldProcessingService { + + private final AttributesService attributesService; + private final TimeseriesService timeseriesService; + private final TbClusterService clusterService; + private final ApiLimitService apiLimitService; + private final PartitionService partitionService; + + private ListeningExecutorService calculatedFieldCallbackExecutor; + + @PostConstruct + public void init() { + calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); + } + + @PreDestroy + public void stop() { + if (calculatedFieldCallbackExecutor != null) { + calculatedFieldCallbackExecutor.shutdownNow(); + } + } + + @Override + public ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId) { + Map> argFutures = new HashMap<>(); + for (var entry : ctx.getArguments().entrySet()) { + var argEntityId = entry.getValue().getRefEntityId() != null ? entry.getValue().getRefEntityId() : entityId; + var argValueFuture = fetchKvEntry(ctx.getTenantId(), argEntityId, entry.getValue()); + argFutures.put(entry.getKey(), argValueFuture); + } + return Futures.whenAllComplete(argFutures.values()).call(() -> { + var result = createStateByType(ctx); + result.updateState(argFutures.entrySet().stream() + .collect(Collectors.toMap( + Entry::getKey, // Keep the key as is + entry -> { + try { + // Resolve the future to get the value + return entry.getValue().get(); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException("Error getting future result for key: " + entry.getKey(), e); + } + } + ))); + return result; + }, calculatedFieldCallbackExecutor); + } + + @Override + public Map fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map arguments) { + Map> argFutures = new HashMap<>(); + for (var entry : arguments.entrySet()) { + var argEntityId = entry.getValue().getRefEntityId() != null ? entry.getValue().getRefEntityId() : entityId; + var argValueFuture = fetchKvEntry(tenantId, argEntityId, entry.getValue()); + argFutures.put(entry.getKey(), argValueFuture); + } + return argFutures.entrySet().stream() + .collect(Collectors.toMap( + Entry::getKey, // Keep the key as is + entry -> { + try { + // Resolve the future to get the value + return entry.getValue().get(); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException("Error getting future result for key: " + entry.getKey(), e); + } + } + )); + } + + @Override + public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculatedFieldResult, List cfIds, TbCallback callback) { + try { + OutputType type = calculatedFieldResult.getType(); + TbMsgType msgType = OutputType.ATTRIBUTES.equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; + TbMsgMetaData md = OutputType.ATTRIBUTES.equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; + TbMsg msg = TbMsg.newMsg().type(msgType).originator(entityId).previousCalculatedFieldIds(cfIds).metaData(md).data(JacksonUtil.writeValueAsString(calculatedFieldResult.getResult())).build(); + clusterService.pushMsgToRuleEngine(tenantId, entityId, msg, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + callback.onSuccess(); + log.trace("[{}][{}] Pushed message to rule engine: {} ", tenantId, entityId, msg); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + }); + } catch (Exception e) { + log.warn("[{}][{}] Failed to push message to rule engine. CalculatedFieldResult: {}", tenantId, entityId, calculatedFieldResult, e); + callback.onFailure(e); + } + } + + @Override + public void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List linkedCalculatedFields, TbCallback callback) { + Map> unicasts = new HashMap<>(); + List broadcasts = new ArrayList<>(); + for (CalculatedFieldEntityCtxId link : linkedCalculatedFields) { + var linkEntityId = link.entityId(); + var linkEntityType = linkEntityId.getEntityType(); + // Let's assume number of entities in profile is N, and number of partitions is P. If N > P, we save by broadcasting to all partitions. Usually N >> P. + boolean broadcast = EntityType.DEVICE_PROFILE.equals(linkEntityType) || EntityType.ASSET_PROFILE.equals(linkEntityType); + if (broadcast) { + broadcasts.add(link); + } else { + TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF, link.entityId()); + unicasts.computeIfAbsent(tpi, k -> new ArrayList<>()).add(link); + } + } + MultipleTbCallback linkCallback = new MultipleTbCallback(2, callback); + if (!broadcasts.isEmpty()) { + broadcast(broadcasts, msg, linkCallback); + } else { + linkCallback.onSuccess(); + } + if (!unicasts.isEmpty()) { + unicast(unicasts, msg, linkCallback); + } else { + linkCallback.onSuccess(); + } + } + + private void unicast(Map> unicasts, CalculatedFieldTelemetryMsg msg, MultipleTbCallback mainCallback) { + TbQueueCallback callback = new TbCallbackWrapper(new MultipleTbCallback(unicasts.size(), mainCallback)); + unicasts.forEach((topicPartitionInfo, ctxIds) -> { + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(msg.getProto(), ctxIds); + clusterService.pushMsgToCalculatedFields(topicPartitionInfo, UUID.randomUUID(), + ToCalculatedFieldMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), callback); + }); + } + + private void broadcast(List broadcasts, CalculatedFieldTelemetryMsg msg, MultipleTbCallback mainCallback) { + TbQueueCallback callback = new TbCallbackWrapper(mainCallback); + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(msg.getProto(), broadcasts); + clusterService.broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), callback); + } + + private CalculatedFieldLinkedTelemetryMsgProto buildLinkedTelemetryMsgProto(CalculatedFieldTelemetryMsgProto telemetryProto, List links) { + Builder builder = CalculatedFieldLinkedTelemetryMsgProto.newBuilder(); + builder.setMsg(telemetryProto); + for (CalculatedFieldEntityCtxId link : links) { + builder.addLinks(toProto(link)); + } + return builder.build(); + } + + private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { + return switch (argument.getRefEntityKey().getType()) { + case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument); + case ATTRIBUTE -> transformSingleValueArgument( + Futures.transform( + attributesService.find(tenantId, entityId, argument.getRefEntityKey().getScope(), argument.getRefEntityKey().getKey()), + result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(createDefaultKvEntry(argument), System.currentTimeMillis(), 0L))), + calculatedFieldCallbackExecutor) + ); + case TS_LATEST -> transformSingleValueArgument( + Futures.transform( + timeseriesService.findLatest(tenantId, entityId, argument.getRefEntityKey().getKey()), + result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument), 0L))), + calculatedFieldCallbackExecutor)); + }; + } + + private ListenableFuture transformSingleValueArgument(ListenableFuture> kvEntryFuture) { + return Futures.transform(kvEntryFuture, kvEntry -> { + if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { + return ArgumentEntry.createSingleValueArgument(kvEntry.get()); + } else { + return new SingleValueArgumentEntry(); + } + }, calculatedFieldCallbackExecutor); + } + + private ListenableFuture fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument) { + long currentTime = System.currentTimeMillis(); + long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); + long startTs = currentTime - timeWindow; + long maxDataPoints = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); + int limit = argument.getLimit() == 0 ? (int) maxDataPoints : argument.getLimit(); + + ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); + ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); + + return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? new TsRollingArgumentEntry(limit, timeWindow) : ArgumentEntry.createTsRollingArgument(tsRolling, limit, timeWindow), calculatedFieldCallbackExecutor); + } + + private KvEntry createDefaultKvEntry(Argument argument) { + String key = argument.getRefEntityKey().getKey(); + String defaultValue = argument.getDefaultValue(); + if (StringUtils.isBlank(defaultValue)) { + return new StringDataEntry(key, null); + } + if (NumberUtils.isParsable(defaultValue)) { + return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); + } + if ("true".equalsIgnoreCase(defaultValue) || "false".equalsIgnoreCase(defaultValue)) { + return new BooleanDataEntry(key, Boolean.parseBoolean(defaultValue)); + } + return new StringDataEntry(key, defaultValue); + } + + private CalculatedFieldState createStateByType(CalculatedFieldCtx ctx) { + return switch (ctx.getCfType()) { + case SIMPLE -> new SimpleCalculatedFieldState(ctx.getArgNames()); + case SCRIPT -> new ScriptCalculatedFieldState(ctx.getArgNames()); + }; + } + + private static class TbCallbackWrapper implements TbQueueCallback { + private final TbCallback callback; + + public TbCallbackWrapper(TbCallback callback) { + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + callback.onSuccess(); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java new file mode 100644 index 0000000000..3ca0b220ff --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -0,0 +1,258 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import com.google.common.util.concurrent.FutureCallback; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.rule.engine.api.AttributesDeleteRequest; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; +import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; + +import java.util.EnumSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.function.Predicate; +import java.util.function.Supplier; + +import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; +import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; + +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueService { + + public static final TbQueueCallback DUMMY_TB_QUEUE_CALLBACK = new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + } + + @Override + public void onFailure(Throwable t) { + } + }; + + private final TbAssetProfileCache assetProfileCache; + private final TbDeviceProfileCache deviceProfileCache; + private final CalculatedFieldCache calculatedFieldCache; + private final TbClusterService clusterService; + + private static final Set supportedReferencedEntities = EnumSet.of( + EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT + ); + + @Override + public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback) { + var tenantId = request.getTenantId(); + var entityId = request.getEntityId(); + checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries()), cf -> cf.linkMatches(entityId, request.getEntries()), + () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); + } + + @Override + public void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback) { + var tenantId = request.getTenantId(); + var entityId = request.getEntityId(); + checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), + () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); + } + + @Override + public void pushRequestToQueue(AttributesDeleteRequest request, List result, FutureCallback callback) { + var tenantId = request.getTenantId(); + var entityId = request.getEntityId(); + checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matchesKeys(result, request.getScope()), cf -> cf.linkMatchesAttrKeys(entityId, result, request.getScope()), + () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); + } + + @Override + public void pushRequestToQueue(TimeseriesDeleteRequest request, List result, FutureCallback callback) { + var tenantId = request.getTenantId(); + var entityId = request.getEntityId(); + + checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matchesKeys(result), cf -> cf.linkMatchesTsKeys(entityId, result), + () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); + } + + private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, + Predicate mainEntityFilter, Predicate linkedEntityFilter, + Supplier msg, FutureCallback callback) { + boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); + if (send) { + clusterService.pushMsgToCalculatedFields(tenantId, entityId, msg.get(), wrap(callback)); + } else { + if (callback != null) { + callback.onSuccess(null); + } + } + } + + private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter) { + boolean send = false; + if (supportedReferencedEntities.contains(entityId.getEntityType())) { + send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(entityId).stream().anyMatch(filter); + if (!send) { + send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(getProfileId(tenantId, entityId)).stream().anyMatch(filter); + } + if (!send) { + send = calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId).stream() + .map(CalculatedFieldLink::getCalculatedFieldId) + .map(calculatedFieldCache::getCalculatedFieldCtx) + .anyMatch(linkedEntityFilter); + } + } + return send; + } + + private EntityId getProfileId(TenantId tenantId, EntityId entityId) { + return switch (entityId.getEntityType()) { + case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); + default -> null; + }; + } + + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { + ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); + + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); + List entries = request.getEntries(); + List versions = result.getVersions(); + for (int i = 0; i < entries.size(); i++) { + long tsVersion = versions.get(i); + TsKvProto tsProto = toTsKvProto(entries.get(i)).toBuilder().setVersion(tsVersion).build(); + telemetryMsg.addTsData(tsProto); + } + msg.setTelemetryMsg(telemetryMsg.build()); + + return msg.build(); + } + + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List versions) { + ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); + + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); + telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); + List entries = request.getEntries(); + for (int i = 0; i < entries.size(); i++) { + long attrVersion = versions.get(i); + AttributeValueProto attrProto = ProtoUtils.toProto(entries.get(i)).toBuilder().setVersion(attrVersion).build(); + telemetryMsg.addAttrData(attrProto); + } + msg.setTelemetryMsg(telemetryMsg.build()); + + return msg.build(); + } + + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesDeleteRequest request, List removedKeys) { + CalculatedFieldTelemetryMsgProto telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()) + .setScope(AttributeScopeProto.valueOf(request.getScope().name())) + .addAllRemovedAttrKeys(removedKeys).build(); + return ToCalculatedFieldMsg.newBuilder() + .setTelemetryMsg(telemetryMsg) + .build(); + } + + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesDeleteRequest request, List removedKeys) { + CalculatedFieldTelemetryMsgProto telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()) + .addAllRemovedTsKeys(removedKeys).build(); + return ToCalculatedFieldMsg.newBuilder() + .setTelemetryMsg(telemetryMsg) + .build(); + } + + private CalculatedFieldTelemetryMsgProto.Builder buildTelemetryMsgProto(TenantId tenantId, EntityId entityId, List calculatedFieldIds, UUID tbMsgId, TbMsgType tbMsgType) { + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = CalculatedFieldTelemetryMsgProto.newBuilder(); + + telemetryMsg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + telemetryMsg.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + + telemetryMsg.setEntityType(entityId.getEntityType().name()); + telemetryMsg.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + + if (calculatedFieldIds != null) { + for (CalculatedFieldId cfId : calculatedFieldIds) { + telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); + } + } + + if (tbMsgId != null) { + telemetryMsg.setTbMsgIdMSB(tbMsgId.getMostSignificantBits()); + telemetryMsg.setTbMsgIdLSB(tbMsgId.getLeastSignificantBits()); + } + + if (tbMsgType != null) { + telemetryMsg.setTbMsgType(tbMsgType.name()); + } + + return telemetryMsg; + } + + private static TbQueueCallback wrap(FutureCallback callback) { + if (callback != null) { + return new FutureCallbackWrapper(callback); + } else { + return DUMMY_TB_QUEUE_CALLBACK; + } + } + + private static class FutureCallbackWrapper implements TbQueueCallback { + private final FutureCallback callback; + + public FutureCallbackWrapper(FutureCallback callback) { + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + callback.onSuccess(null); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java new file mode 100644 index 0000000000..bb5ef91974 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.cache; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; + +import java.util.Collection; + +public interface CalculatedFieldEntityProfileCache extends ApplicationListener { + + void add(TenantId tenantId, EntityId profileId, EntityId entityId); + + void update(TenantId tenantId, EntityId oldProfileId, EntityId newProfileId, EntityId entityId); + + void evict(TenantId tenantId, EntityId entityId); + + Collection getMyEntityIdsByProfileId(TenantId tenantId, EntityId profileId); + + int getEntityIdPartition(TenantId tenantId, EntityId entityId); +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java new file mode 100644 index 0000000000..4cf62c01b3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java @@ -0,0 +1,93 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.cache; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.TbApplicationEventListener; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; + +@TbRuleEngineComponent +@Service +@Slf4j +@RequiredArgsConstructor +//TODO ashvayka: remove and use TenantEntityProfileCache in each CalculatedFieldManagerMessageProcessor; +public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEventListener implements CalculatedFieldEntityProfileCache { + + private static final Integer UNKNOWN = 0; + private final ConcurrentMap tenantCache = new ConcurrentHashMap<>(); + private final PartitionService partitionService; + private volatile List myPartitions = Collections.emptyList(); + + @Override + protected void onTbApplicationEvent(PartitionChangeEvent event) { + myPartitions = event.getCfPartitions().stream() + .filter(TopicPartitionInfo::isMyPartition) + .map(tpi -> tpi.getPartition().orElse(UNKNOWN)).collect(Collectors.toList()); + //Naive approach that need to be improved. + tenantCache.values().forEach(cache -> cache.setMyPartitions(myPartitions)); + } + + @Override + public void add(TenantId tenantId, EntityId profileId, EntityId entityId) { + var tpi = partitionService.resolve(QueueKey.CF, entityId); + var partition = tpi.getPartition().orElse(UNKNOWN); + tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()) + .add(profileId, entityId, partition, tpi.isMyPartition()); + } + + @Override + public void update(TenantId tenantId, EntityId oldProfileId, EntityId newProfileId, EntityId entityId) { + var tpi = partitionService.resolve(QueueKey.CF, entityId); + var partition = tpi.getPartition().orElse(UNKNOWN); + var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); + //TODO: make this method atomic; + cache.remove(oldProfileId, entityId); + cache.add(newProfileId, entityId, partition, tpi.isMyPartition()); + } + + @Override + public void evict(TenantId tenantId, EntityId entityId) { + var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); + cache.removeEntityId(entityId); + } + + @Override + public Collection getMyEntityIdsByProfileId(TenantId tenantId, EntityId profileId) { + return tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()).getMyEntityIdsByProfileId(profileId); + } + + @Override + public int getEntityIdPartition(TenantId tenantId, EntityId entityId) { + var tpi = partitionService.resolve(QueueKey.CF, entityId); + return tpi.getPartition().orElse(UNKNOWN); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java new file mode 100644 index 0000000000..1a17b9b8be --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java @@ -0,0 +1,122 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.cache; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class TenantEntityProfileCache { + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private final Map>> allEntities = new HashMap<>(); + private final Map> myEntities = new HashMap<>(); + + public void setMyPartitions(List myPartitions) { + lock.writeLock().lock(); + try { + myEntities.clear(); + myPartitions.forEach(partitionId -> { + var map = allEntities.get(partitionId); + if (map != null) { + map.forEach((profileId, entityIds) -> myEntities.computeIfAbsent(profileId, k -> new HashSet<>()).addAll(entityIds)); + } + }); + } finally { + lock.writeLock().unlock(); + } + } + + public void removeProfileId(EntityId profileId) { + lock.writeLock().lock(); + try { + // Remove from allEntities + allEntities.values().forEach(map -> map.remove(profileId)); + // Remove from myEntities + myEntities.remove(profileId); + } finally { + lock.writeLock().unlock(); + } + } + + public void removeEntityId(EntityId entityId) { + lock.writeLock().lock(); + try { + // Remove from allEntities + allEntities.values().forEach(map -> map.values().forEach(set -> set.remove(entityId))); + // Remove from myEntities + myEntities.values().forEach(set -> set.remove(entityId)); + } finally { + lock.writeLock().unlock(); + } + } + + public void remove(EntityId profileId, EntityId entityId) { + lock.writeLock().lock(); + try { + // Remove from allEntities + allEntities.values().forEach(map -> removeSafely(map, profileId, entityId)); + // Remove from myEntities + removeSafely(myEntities, profileId, entityId); + } finally { + lock.writeLock().unlock(); + } + } + + public void add(EntityId profileId, EntityId entityId, Integer partition, boolean mine) { + lock.writeLock().lock(); + try { + if(EntityType.DEVICE.equals(profileId.getEntityType())){ + throw new RuntimeException("WTF?"); + } + if (mine) { + myEntities.computeIfAbsent(profileId, k -> new HashSet<>()).add(entityId); + } + allEntities.computeIfAbsent(partition, k -> new HashMap<>()).computeIfAbsent(profileId, p -> new HashSet<>()).add(entityId); + } finally { + lock.writeLock().unlock(); + } + } + + public Collection getMyEntityIdsByProfileId(EntityId profileId) { + lock.readLock().lock(); + try { + var entities = myEntities.getOrDefault(profileId, Collections.emptySet()); + List result = new ArrayList<>(entities.size()); + result.addAll(entities); + return result; + } finally { + lock.readLock().unlock(); + } + } + + private void removeSafely(Map> map, EntityId profileId, EntityId entityId) { + var set = map.get(profileId); + if (set != null) { + set.remove(entityId); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java new file mode 100644 index 0000000000..6694252652 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx; + +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; + +@Data +@NoArgsConstructor +public class CalculatedFieldEntityCtx { + + private CalculatedFieldEntityCtxId id; + private CalculatedFieldState state; + + public CalculatedFieldEntityCtx(CalculatedFieldEntityCtxId id, CalculatedFieldState state) { + this.id = id; + this.state = state; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java new file mode 100644 index 0000000000..329028eda4 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx; + +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +public record CalculatedFieldEntityCtxId(TenantId tenantId, CalculatedFieldId cfId, EntityId entityId) { + + public String toKey() { + return cfId + "_" + entityId; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java new file mode 100644 index 0000000000..83e10b8194 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = SingleValueArgumentEntry.class, name = "SINGLE_VALUE"), + @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING") +}) +public interface ArgumentEntry { + + @JsonIgnore + ArgumentEntryType getType(); + + Object getValue(); + + boolean updateEntry(ArgumentEntry entry); + + boolean isEmpty(); + + TbelCfArg toTbelCfArg(); + + boolean isForceResetPrevious(); + + void setForceResetPrevious(boolean forceResetPrevious); + + static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { + return new SingleValueArgumentEntry(kvEntry); + } + + static ArgumentEntry createTsRollingArgument(List kvEntries, int limit, long timeWindow) { + return new TsRollingArgumentEntry(kvEntries, limit, timeWindow); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java new file mode 100644 index 0000000000..68f973c7c1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +public enum ArgumentEntryType { + SINGLE_VALUE, TS_ROLLING +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java new file mode 100644 index 0000000000..d59a58f296 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -0,0 +1,86 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.utils.CalculatedFieldUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Data +@AllArgsConstructor +public abstract class BaseCalculatedFieldState implements CalculatedFieldState { + + protected List requiredArguments; + protected Map arguments; + protected boolean sizeExceedsLimit; + + public BaseCalculatedFieldState(List requiredArguments) { + this.requiredArguments = requiredArguments; + this.arguments = new HashMap<>(); + } + + public BaseCalculatedFieldState() { + this(new ArrayList<>(), new HashMap<>(), false); + } + + @Override + public boolean updateState(Map argumentValues) { + if (arguments == null) { + arguments = new HashMap<>(); + } + + boolean stateUpdated = false; + + for (Map.Entry entry : argumentValues.entrySet()) { + String key = entry.getKey(); + ArgumentEntry newEntry = entry.getValue(); + ArgumentEntry existingEntry = arguments.get(key); + + if (existingEntry == null || newEntry.isForceResetPrevious()) { + validateNewEntry(newEntry); + arguments.put(key, newEntry); + stateUpdated = true; + } else { + stateUpdated = existingEntry.updateEntry(newEntry); + } + } + + return stateUpdated; + } + + @Override + public boolean isReady() { + return arguments.keySet().containsAll(requiredArguments) && + arguments.values().stream().noneMatch(ArgumentEntry::isEmpty); + } + + @Override + public void checkStateSize(CalculatedFieldEntityCtxId ctxId, long maxStateSize) { + if (!sizeExceedsLimit && maxStateSize > 0 && CalculatedFieldUtils.toProto(ctxId, this).getSerializedSize() > maxStateSize) { + arguments.clear(); + sizeExceedsLimit = true; + } + } + + protected abstract void validateNewEntry(ArgumentEntry newEntry); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java new file mode 100644 index 0000000000..d3b7020c25 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -0,0 +1,277 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.Data; +import net.objecthunter.exp4j.Expression; +import net.objecthunter.exp4j.ExpressionBuilder; +import org.mvel2.MVEL; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Data +public class CalculatedFieldCtx { + + private CalculatedField calculatedField; + + private CalculatedFieldId cfId; + private TenantId tenantId; + private EntityId entityId; + private CalculatedFieldType cfType; + private final Map arguments; + private final Map mainEntityArguments; + private final Map> linkedEntityArguments; + + private final Map, String> referencedEntityKeys; + private final List argNames; + private Output output; + private String expression; + private TbelInvokeService tbelInvokeService; + private CalculatedFieldScriptEngine calculatedFieldScriptEngine; + private ThreadLocal customExpression; + + private boolean initialized; + + private long maxDataPointsPerRollingArg; + private long maxStateSize; + + public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService, ApiLimitService apiLimitService) { + this.calculatedField = calculatedField; + + this.cfId = calculatedField.getId(); + this.tenantId = calculatedField.getTenantId(); + this.entityId = calculatedField.getEntityId(); + this.cfType = calculatedField.getType(); + CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); + this.arguments = configuration.getArguments(); + this.mainEntityArguments = new HashMap<>(); + this.linkedEntityArguments = new HashMap<>(); + for (Map.Entry entry : arguments.entrySet()) { + var refId = entry.getValue().getRefEntityId(); + var refKey = entry.getValue().getRefEntityKey(); + if (refId == null) { + mainEntityArguments.put(refKey, entry.getKey()); + } else { + linkedEntityArguments.computeIfAbsent(refId, key -> new HashMap<>()).put(refKey, entry.getKey()); + } + } + this.referencedEntityKeys = arguments.entrySet().stream() + .collect(Collectors.toMap( + entry -> new TbPair<>(entry.getValue().getRefEntityId() == null ? entityId : entry.getValue().getRefEntityId(), entry.getValue().getRefEntityKey()), + Map.Entry::getKey + )); + this.argNames = new ArrayList<>(arguments.keySet()); + this.output = configuration.getOutput(); + this.expression = configuration.getExpression(); + this.tbelInvokeService = tbelInvokeService; + + this.maxDataPointsPerRollingArg = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); + this.maxStateSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes) * 1024; + } + + public void init() { + if (CalculatedFieldType.SCRIPT.equals(cfType)) { + try { + this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); + initialized = true; + } catch (Exception e) { + throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax.", e); + } + } else { + if (isValidExpression(expression)) { + this.customExpression = ThreadLocal.withInitial(() -> + new ExpressionBuilder(expression) + .implicitMultiplication(true) + .variables(this.arguments.keySet()) + .build() + ); + initialized = true; + } else { + throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax."); + } + } + } + + private CalculatedFieldScriptEngine initEngine(TenantId tenantId, String expression, TbelInvokeService tbelInvokeService) { + if (tbelInvokeService == null) { + throw new IllegalArgumentException("TBEL script engine is disabled!"); + } + + return new CalculatedFieldTbelScriptEngine( + tenantId, + tbelInvokeService, + expression, + argNames.toArray(String[]::new) + ); + } + + private boolean isValidExpression(String expression) { + try { + MVEL.compileExpression(expression); + return true; + } catch (Exception e) { + return false; + } + } + + public boolean matches(List values, AttributeScope scope) { + return matchesAttributes(mainEntityArguments, values, scope); + } + + public boolean linkMatches(EntityId entityId, List values, AttributeScope scope) { + var map = linkedEntityArguments.get(entityId); + return map != null && matchesAttributes(map, values, scope); + } + + public boolean matches(List values) { + return matchesTimeSeries(mainEntityArguments, values); + } + + public boolean linkMatches(EntityId entityId, List values) { + var map = linkedEntityArguments.get(entityId); + return map != null && matchesTimeSeries(map, values); + } + + private boolean matchesAttributes(Map argMap, List values, AttributeScope scope) { + for (AttributeKvEntry attrKv : values) { + ReferencedEntityKey attrKey = new ReferencedEntityKey(attrKv.getKey(), ArgumentType.ATTRIBUTE, scope); + if (argMap.containsKey(attrKey)) { + return true; + } + } + return false; + } + + private boolean matchesTimeSeries(Map argMap, List values) { + for (TsKvEntry tsKv : values) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_LATEST, null); + if (argMap.containsKey(latestKey)) { + return true; + } + ReferencedEntityKey rollingKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_ROLLING, null); + if (argMap.containsKey(rollingKey)) { + return true; + } + } + return false; + } + + public boolean matchesKeys(List keys, AttributeScope scope) { + return matchesAttributesKeys(mainEntityArguments, keys, scope); + } + + public boolean matchesKeys(List keys) { + return matchesTimeSeriesKeys(mainEntityArguments, keys); + } + + private boolean matchesAttributesKeys(Map argMap, List keys, AttributeScope scope) { + for (String key : keys) { + ReferencedEntityKey attrKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); + if (argMap.containsKey(attrKey)) { + return true; + } + } + return false; + } + + private boolean matchesTimeSeriesKeys(Map argMap, List keys) { + for (String key : keys) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); + if (argMap.containsKey(latestKey)) { + return true; + } + ReferencedEntityKey rollingKey = new ReferencedEntityKey(key, ArgumentType.TS_ROLLING, null); + if (argMap.containsKey(rollingKey)) { + return true; + } + } + return false; + } + + public boolean linkMatchesAttrKeys(EntityId entityId, List keys, AttributeScope scope) { + var map = linkedEntityArguments.get(entityId); + return map != null && matchesAttributesKeys(map, keys, scope); + } + + public boolean linkMatchesTsKeys(EntityId entityId, List keys) { + var map = linkedEntityArguments.get(entityId); + return map != null && matchesTimeSeriesKeys(map, keys); + } + + public boolean linkMatches(EntityId entityId, CalculatedFieldTelemetryMsgProto proto) { + if (!proto.getTsDataList().isEmpty()) { + List updatedTelemetry = proto.getTsDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return linkMatches(entityId, updatedTelemetry); + } else if (!proto.getAttrDataList().isEmpty()) { + AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); + List updatedTelemetry = proto.getAttrDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return linkMatches(entityId, updatedTelemetry, scope); + } else if (!proto.getRemovedTsKeysList().isEmpty()) { + return linkMatchesTsKeys(entityId, proto.getRemovedTsKeysList()); + } else { + return linkMatchesAttrKeys(entityId, proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); + } + } + + public CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId() { + return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); + } + + public boolean hasOtherSignificantChanges(CalculatedFieldCtx other) { + boolean expressionChanged = !expression.equals(other.expression); + boolean outputChanged = !output.equals(other.output); + return expressionChanged || outputChanged; + } + + public boolean hasStateChanges(CalculatedFieldCtx other) { + boolean typeChanged = !cfType.equals(other.cfType); + boolean argumentsChanged = !arguments.equals(other.arguments); + return typeChanged || argumentsChanged; + } + + public String getSizeExceedsLimitMessage() { + return "Failed to init CF state. State size exceeds limit of " + (maxStateSize / 1024) + "Kb!"; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java new file mode 100644 index 0000000000..caad1e4cfe --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.util.concurrent.ListenableFuture; + +public interface CalculatedFieldScriptEngine { + + ListenableFuture executeScriptAsync(Object[] args); + + ListenableFuture executeJsonAsync(Object[] args); + + void destroy(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java new file mode 100644 index 0000000000..70587d8b1e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.service.cf.CalculatedFieldResult; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; + +import java.util.List; +import java.util.Map; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE"), + @JsonSubTypes.Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT"), +}) +public interface CalculatedFieldState { + + @JsonIgnore + CalculatedFieldType getType(); + + Map getArguments(); + + void setRequiredArguments(List requiredArguments); + + boolean updateState(Map argumentValues); + + ListenableFuture performCalculation(CalculatedFieldCtx ctx); + + @JsonIgnore + boolean isReady(); + + boolean isSizeExceedsLimit(); + + @JsonIgnore + default boolean isSizeOk() { + return !isSizeExceedsLimit(); + } + + void checkStateSize(CalculatedFieldEntityCtxId ctxId, long maxStateSize); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java new file mode 100644 index 0000000000..2ca34b7b17 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -0,0 +1,82 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.ScriptType; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.id.TenantId; + +import javax.script.ScriptException; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +@Slf4j +public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEngine { + + private final TbelInvokeService tbelInvokeService; + + private final UUID scriptId; + private final TenantId tenantId; + + public CalculatedFieldTbelScriptEngine(TenantId tenantId, TbelInvokeService tbelInvokeService, String script, String... argNames) { + this.tenantId = tenantId; + this.tbelInvokeService = tbelInvokeService; + try { + this.scriptId = this.tbelInvokeService.eval(tenantId, ScriptType.CALCULATED_FIELD_SCRIPT, script, argNames).get(); + } catch (Exception e) { + Throwable t = e; + if (e instanceof ExecutionException) { + t = e.getCause(); + } + throw new IllegalArgumentException("Can't compile script: " + t.getMessage(), t); + } + } + + @Override + public ListenableFuture executeScriptAsync(Object[] args) { + log.trace("Executing script async, args {}", args); + return Futures.transformAsync(tbelInvokeService.invokeScript(tenantId, null, this.scriptId, args), + o -> { + try { + return Futures.immediateFuture(o); + } catch (Exception e) { + if (e.getCause() instanceof ScriptException) { + return Futures.immediateFailedFuture(e.getCause()); + } else if (e.getCause() instanceof RuntimeException) { + return Futures.immediateFailedFuture(new ScriptException(e.getCause().getMessage())); + } else { + return Futures.immediateFailedFuture(new ScriptException(e)); + } + } + }, MoreExecutors.directExecutor()); + } + + @Override + public ListenableFuture executeJsonAsync(Object[] args) { + return Futures.transform(executeScriptAsync(args), JacksonUtil::valueToTree, MoreExecutors.directExecutor()); + } + + @Override + public void destroy() { + tbelInvokeService.release(this.scriptId); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java new file mode 100644 index 0000000000..e81fa4d1dc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -0,0 +1,157 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgHeaders; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; +import org.thingsboard.server.queue.common.consumer.QueueStateService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; +import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; + +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.*; + +@Service +@RequiredArgsConstructor +@Slf4j +@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine') && '${queue.type:null}'=='kafka'") +public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldStateService { + + private final TbRuleEngineQueueFactory queueFactory; + private final PartitionService partitionService; + + @Value("${queue.calculated_fields.poll_interval:25}") + private long pollInterval; + + private PartitionedQueueConsumerManager> stateConsumer; + private TbKafkaProducerTemplate> stateProducer; + private QueueStateService, TbProtoQueueMsg> queueStateService; + + private final AtomicInteger counter = new AtomicInteger(); + + @Override + public void init(PartitionedQueueConsumerManager> eventConsumer) { + super.init(eventConsumer); + this.stateConsumer = PartitionedQueueConsumerManager.>create() + .queueKey(QueueKey.CF_STATES) + .topic(partitionService.getTopic(QueueKey.CF_STATES)) + .pollInterval(pollInterval) + .msgPackProcessor((msgs, consumer, config) -> { + for (TbProtoQueueMsg msg : msgs) { + try { + if (msg.getValue() != null) { + processRestoredState(msg.getValue()); + } else { + processRestoredState(getStateId(msg.getHeaders()), null); + } + } catch (Throwable t) { + log.error("Failed to process state message: {}", msg, t); + } + + int processedMsgCount = counter.incrementAndGet(); + if (processedMsgCount % 10000 == 0) { + log.info("Processed {} calculated field state msgs", processedMsgCount); + } + } + }) + .consumerCreator((config, partitionId) -> queueFactory.createCalculatedFieldStateConsumer()) + .consumerExecutor(eventConsumer.getConsumerExecutor()) + .scheduler(eventConsumer.getScheduler()) + .taskExecutor(eventConsumer.getTaskExecutor()) + .build(); + this.stateProducer = (TbKafkaProducerTemplate>) queueFactory.createCalculatedFieldStateProducer(); + this.queueStateService = new QueueStateService<>(); + this.queueStateService.init(stateConsumer, super.eventConsumer); + } + + @Override + protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF_STATES, stateId.entityId()); + TbProtoQueueMsg msg = new TbProtoQueueMsg<>(stateId.entityId().getId(), stateMsgProto); + if (stateMsgProto == null) { + putStateId(msg.getHeaders(), stateId); + } + stateProducer.send(tpi, stateId.toKey(), msg, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (callback != null) { + callback.onSuccess(); + } + } + + @Override + public void onFailure(Throwable t) { + if (callback != null) { + callback.onFailure(t); + } + } + }); + } + + @Override + protected void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback) { + doPersist(stateId, null, callback); + } + + @Override + public void restore(Set partitions) { + queueStateService.update(partitions); + } + + private void putStateId(TbQueueMsgHeaders headers, CalculatedFieldEntityCtxId stateId) { + headers.put("tenantId", uuidToBytes(stateId.tenantId().getId())); + headers.put("cfId", uuidToBytes(stateId.cfId().getId())); + headers.put("entityId", uuidToBytes(stateId.entityId().getId())); + headers.put("entityType", stringToBytes(stateId.entityId().getEntityType().name())); + } + + private CalculatedFieldEntityCtxId getStateId(TbQueueMsgHeaders headers) { + TenantId tenantId = TenantId.fromUUID(bytesToUuid(headers.get("tenantId"))); + CalculatedFieldId cfId = new CalculatedFieldId(bytesToUuid(headers.get("cfId"))); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(bytesToString(headers.get("entityType")), bytesToUuid(headers.get("entityId"))); + return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); + } + + @Override + public void stop() { + stateConsumer.stop(); + stateConsumer.awaitStop(); + stateProducer.stop(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java new file mode 100644 index 0000000000..a508eecada --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java @@ -0,0 +1,73 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService; +import org.thingsboard.server.service.cf.CfRocksDb; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; + +import java.util.Set; + +@Service +@RequiredArgsConstructor +@Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='in-memory'") +public class RocksDBCalculatedFieldStateService extends AbstractCalculatedFieldStateService { + + private final CfRocksDb cfRocksDb; + + private boolean initialized; + + @Override + protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback) { + cfRocksDb.put(stateId.toKey(), stateMsgProto.toByteArray()); + callback.onSuccess(); + } + + @Override + protected void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback) { + cfRocksDb.delete(stateId.toKey()); + callback.onSuccess(); + } + + @Override + public void restore(Set partitions) { + if (!this.initialized) { + cfRocksDb.forEach((key, value) -> { + try { + processRestoredState(CalculatedFieldStateProto.parseFrom(value)); + } catch (InvalidProtocolBufferException e) { + log.error("[{}] Failed to process restored state", key, e); + } + }); + this.initialized = true; + } + eventConsumer.update(partitions); + } + + @Override + public void stop() { + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java new file mode 100644 index 0000000000..00d9ccbe5a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -0,0 +1,68 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.service.cf.CalculatedFieldResult; + +import java.util.List; + +@Data +@Slf4j +@NoArgsConstructor +public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { + + public ScriptCalculatedFieldState(List requiredArguments) { + super(requiredArguments); + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SCRIPT; + } + + @Override + protected void validateNewEntry(ArgumentEntry newEntry) { + } + + @Override + public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { + Object[] args = ctx.getArgNames().stream() + .map(this::toTbelArgument) + .toArray(); + + ListenableFuture resultFuture = ctx.getCalculatedFieldScriptEngine().executeJsonAsync(args); + Output output = ctx.getOutput(); + return Futures.transform(resultFuture, + result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), + MoreExecutors.directExecutor() + ); + } + + private TbelCfArg toTbelArgument(String key) { + return arguments.get(key).toTbelCfArg(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java new file mode 100644 index 0000000000..25008a60bd --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -0,0 +1,70 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.service.cf.CalculatedFieldResult; + +import java.util.List; +import java.util.Map; + +@Data +@NoArgsConstructor +public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { + + public SimpleCalculatedFieldState(List requiredArguments) { + super(requiredArguments); + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SIMPLE; + } + + @Override + protected void validateNewEntry(ArgumentEntry newEntry) { + if (newEntry instanceof TsRollingArgumentEntry) { + throw new IllegalArgumentException("Rolling argument entry is not supported for simple calculated fields."); + } + } + + @Override + public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { + var expr = ctx.getCustomExpression().get(); + + for (Map.Entry entry : this.arguments.entrySet()) { + try { + BasicKvEntry kvEntry = ((SingleValueArgumentEntry) entry.getValue()).getKvEntryValue(); + expr.setVariable(entry.getKey(), Double.parseDouble(kvEntry.getValueAsString())); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Argument '" + entry.getKey() + "' is not a number."); + } + } + + double expressionResult = expr.evaluate(); + + Output output = ctx.getOutput(); + return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), JacksonUtil.valueToTree(Map.of(output.getName(), expressionResult)))); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java new file mode 100644 index 0000000000..a064a99935 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -0,0 +1,115 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.script.api.tbel.TbelCfSingleValueArg; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class SingleValueArgumentEntry implements ArgumentEntry { + + private long ts; + private BasicKvEntry kvEntryValue; + private Long version; + + private boolean forceResetPrevious; + + public SingleValueArgumentEntry(TsKvProto entry) { + this.ts = entry.getTs(); + this.version = entry.getVersion(); + this.kvEntryValue = ProtoUtils.fromProto(entry.getKv()); + } + + public SingleValueArgumentEntry(AttributeValueProto entry) { + this.ts = entry.getLastUpdateTs(); + this.version = entry.getVersion(); + this.kvEntryValue = ProtoUtils.basicKvEntryFromProto(entry); + } + + public SingleValueArgumentEntry(KvEntry entry) { + if (entry instanceof TsKvEntry tsKvEntry) { + this.ts = tsKvEntry.getTs(); + this.version = tsKvEntry.getVersion(); + } else if (entry instanceof AttributeKvEntry attributeKvEntry) { + this.ts = attributeKvEntry.getLastUpdateTs(); + this.version = attributeKvEntry.getVersion(); + } + this.kvEntryValue = ProtoUtils.basicKvEntryFromKvEntry(entry); + } + + public SingleValueArgumentEntry(long ts, BasicKvEntry kvEntryValue, Long version) { + this.ts = ts; + this.kvEntryValue = kvEntryValue; + this.version = version; + } + + @Override + public ArgumentEntryType getType() { + return ArgumentEntryType.SINGLE_VALUE; + } + + @Override + public boolean isEmpty() { + return kvEntryValue == null; + } + + @JsonIgnore + public Object getValue() { + return isEmpty() ? null : kvEntryValue.getValue(); + } + + @Override + public TbelCfArg toTbelCfArg() { + return new TbelCfSingleValueArg(ts, kvEntryValue.getValue()); + } + + @Override + public boolean updateEntry(ArgumentEntry entry) { + if (entry instanceof SingleValueArgumentEntry singleValueEntry) { + if (singleValueEntry.getTs() == this.ts) { + return false; + } + + Long newVersion = singleValueEntry.getVersion(); + if (newVersion == null || this.version == null || newVersion > this.version) { + this.ts = singleValueEntry.getTs(); + this.version = newVersion; + BasicKvEntry newValue = singleValueEntry.getKvEntryValue(); + if (this.kvEntryValue != null && this.kvEntryValue.getValue().equals(newValue.getValue())) { + return false; + } + this.kvEntryValue = singleValueEntry.getKvEntryValue(); + return true; + } + } else { + throw new IllegalArgumentException("Unsupported argument entry type for single value argument entry: " + entry.getType()); + } + return false; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java new file mode 100644 index 0000000000..1114d993a3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -0,0 +1,146 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.script.api.tbel.TbelCfTsDoubleVal; +import org.thingsboard.script.api.tbel.TbelCfTsRollingArg; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Slf4j +public class TsRollingArgumentEntry implements ArgumentEntry { + + private Integer limit; + private Long timeWindow; + private TreeMap tsRecords = new TreeMap<>(); + + private boolean forceResetPrevious; + + public TsRollingArgumentEntry(List kvEntries, int limit, long timeWindow) { + this.limit = limit; + this.timeWindow = timeWindow; + kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry)); + } + + public TsRollingArgumentEntry(TreeMap tsRecords, int limit, long timeWindow) { + this.tsRecords = tsRecords; + this.limit = limit; + this.timeWindow = timeWindow; + } + + public TsRollingArgumentEntry(int limit, long timeWindow) { + this.tsRecords = new TreeMap<>(); + this.limit = limit; + this.timeWindow = timeWindow; + } + + public TsRollingArgumentEntry(Integer limit, Long timeWindow, TreeMap tsRecords) { + this.limit = limit; + this.timeWindow = timeWindow; + this.tsRecords = tsRecords; + } + + @Override + public ArgumentEntryType getType() { + return ArgumentEntryType.TS_ROLLING; + } + + @Override + public boolean isEmpty() { + return tsRecords.isEmpty(); + } + + @JsonIgnore + @Override + public Object getValue() { + return tsRecords; + } + + @Override + public TbelCfArg toTbelCfArg() { + List values = new ArrayList<>(tsRecords.size()); + for (var e : tsRecords.entrySet()) { + values.add(new TbelCfTsDoubleVal(e.getKey(), e.getValue())); + } + return new TbelCfTsRollingArg(limit, timeWindow, values); + } + + @Override + public boolean updateEntry(ArgumentEntry entry) { + if (entry instanceof TsRollingArgumentEntry tsRollingEntry) { + updateTsRollingEntry(tsRollingEntry); + } else if (entry instanceof SingleValueArgumentEntry singleValueEntry) { + updateSingleValueEntry(singleValueEntry); + } else { + throw new IllegalArgumentException("Unsupported argument entry type for rolling argument entry: " + entry.getType()); + } + return true; + } + + private void updateTsRollingEntry(TsRollingArgumentEntry tsRollingEntry) { + for (Map.Entry tsRecordEntry : tsRollingEntry.getTsRecords().entrySet()) { + addTsRecord(tsRecordEntry.getKey(), tsRecordEntry.getValue()); + } + } + + private void updateSingleValueEntry(SingleValueArgumentEntry singleValueEntry) { + addTsRecord(singleValueEntry.getTs(), singleValueEntry.getKvEntryValue()); + } + + private void addTsRecord(Long ts, KvEntry value) { + try { + switch (value.getDataType()) { + case LONG -> value.getLongValue().ifPresent(aLong -> tsRecords.put(ts, aLong.doubleValue())); + case DOUBLE -> value.getDoubleValue().ifPresent(aDouble -> tsRecords.put(ts, aDouble)); + case BOOLEAN -> value.getBooleanValue().ifPresent(aBoolean -> tsRecords.put(ts, aBoolean ? 1.0 : 0.0)); + case STRING -> value.getStrValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); + case JSON -> value.getJsonValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); + } + } catch (Exception e) { + tsRecords.put(ts, Double.NaN); + log.debug("Invalid value '{}' for time series rolling arguments. Only numeric values are supported.", value.getValue()); + } finally { + cleanupExpiredRecords(); + } + } + + private void addTsRecord(Long ts, double value) { + tsRecords.put(ts, value); + cleanupExpiredRecords(); + } + + private void cleanupExpiredRecords() { + if (tsRecords.size() > limit) { + tsRecords.pollFirstEntry(); + } + tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - timeWindow); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java index 53dbaf0d02..c62a551310 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java @@ -140,6 +140,9 @@ public class EdgeEventSourcingListener { if (EntityType.DEVICE.equals(event.getEntityId().getEntityType()) && ActionType.ASSIGNED_TO_TENANT.equals(event.getActionType())) { return; } + if (EntityType.ALARM.equals(event.getEntityId().getEntityType())) { + return; + } try { if (event.getEntityId().getEntityType().equals(EntityType.RULE_CHAIN) && event.getEdgeId() != null && event.getActionType().equals(ActionType.ASSIGNED_TO_EDGE)) { try { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index aff013ca2b..e9ee7202a7 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -31,9 +31,15 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; +import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; +import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; @@ -71,6 +77,8 @@ public abstract class AbstractTbEntityService { @Autowired(required = false) @Lazy private EntitiesVersionControlService vcService; + @Autowired + protected EntityService entityService; protected boolean isTestProfile() { return Set.of(this.env.getActiveProfiles()).contains("test"); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 8259837e2a..4741275370 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -31,7 +31,9 @@ import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.DeviceId; @@ -54,6 +56,7 @@ import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.TbQueueCallback; import java.util.Set; @@ -88,7 +91,10 @@ public class EntityStateSourcingListener { ComponentLifecycleEvent lifecycleEvent = isCreated ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED; switch (entityType) { - case ASSET, ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE -> { + case ASSET -> { + onAssetUpdate(event.getEntity(), event.getOldEntity()); + } + case ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, lifecycleEvent); } case RULE_CHAIN -> { @@ -123,7 +129,11 @@ public class EntityStateSourcingListener { ApiUsageState apiUsageState = (ApiUsageState) event.getEntity(); tbClusterService.onApiStateChange(apiUsageState, null); } - default -> {} + case CALCULATED_FIELD -> { + onCalculatedFieldUpdate(event.getEntity(), event.getOldEntity()); + } + default -> { + } } } @@ -135,14 +145,18 @@ public class EntityStateSourcingListener { return; } EntityType entityType = entityId.getEntityType(); - if (!tenantId.isSysTenantId() && entityType != EntityType.TENANT && !tenantService.tenantExists(tenantId)) { + if (!tenantId.isSysTenantId() && entityType != EntityType.TENANT && !tenantService.tenantExists(tenantId)) { log.debug("[{}] Ignoring DeleteEntityEvent because tenant does not exist: {}", tenantId, event); return; } log.debug("[{}][{}][{}] Handling entity deletion event: {}", tenantId, entityType, entityId, event); switch (entityType) { - case ASSET, ASSET_PROFILE, EDGE, ENTITY_VIEW, CUSTOMER, NOTIFICATION_RULE -> { + case ASSET -> { + Asset asset = (Asset) event.getEntity(); + tbClusterService.onAssetDeleted(tenantId, asset, null); + } + case ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); } case NOTIFICATION_REQUEST -> { @@ -154,7 +168,8 @@ public class EntityStateSourcingListener { case RULE_CHAIN -> { RuleChain ruleChain = (RuleChain) event.getEntity(); if (RuleChainType.CORE.equals(ruleChain.getType())) { - Set referencingRuleChainIds = JacksonUtil.fromString(event.getBody(), new TypeReference<>() {}); + Set referencingRuleChainIds = JacksonUtil.fromString(event.getBody(), new TypeReference<>() { + }); if (referencingRuleChainIds != null) { referencingRuleChainIds.forEach(referencingRuleChainId -> tbClusterService.broadcastEntityStateChangeEvent(tenantId, referencingRuleChainId, ComponentLifecycleEvent.UPDATED)); @@ -168,11 +183,11 @@ public class EntityStateSourcingListener { } case TENANT_PROFILE -> { TenantProfile tenantProfile = (TenantProfile) event.getEntity(); - tbClusterService.onTenantProfileDelete(tenantProfile, null); + tbClusterService.onTenantProfileDelete(tenantProfile, TbQueueCallback.EMPTY); } case DEVICE -> { Device device = (Device) event.getEntity(); - tbClusterService.onDeviceDeleted(tenantId, device, null); + tbClusterService.onDeviceDeleted(tenantId, device, TbQueueCallback.EMPTY); } case DEVICE_PROFILE -> { DeviceProfile deviceProfile = (DeviceProfile) event.getEntity(); @@ -180,9 +195,14 @@ public class EntityStateSourcingListener { } case TB_RESOURCE -> { TbResourceInfo tbResource = (TbResourceInfo) event.getEntity(); - tbClusterService.onResourceDeleted(tbResource, null); + tbClusterService.onResourceDeleted(tbResource, TbQueueCallback.EMPTY); + } + case CALCULATED_FIELD -> { + CalculatedField calculatedField = (CalculatedField) event.getEntity(); + tbClusterService.onCalculatedFieldDeleted(calculatedField, TbQueueCallback.EMPTY); + } + default -> { } - default -> {} } } @@ -244,6 +264,15 @@ public class EntityStateSourcingListener { tbClusterService.onDeviceUpdated(device, oldDevice); } + private void onAssetUpdate(Object entity, Object oldEntity) { + Asset asset = (Asset) entity; + Asset oldAsset = null; + if (oldEntity instanceof Asset) { + oldAsset = (Asset) oldEntity; + } + tbClusterService.onAssetUpdated(asset, oldAsset); + } + private void onEdgeEvent(TenantId tenantId, EntityId entityId, Object entity, ComponentLifecycleEvent lifecycleEvent) { if (entity instanceof Edge) { tbClusterService.onEdgeStateChangeEvent(new ComponentLifecycleMsg(tenantId, entityId, lifecycleEvent)); @@ -252,6 +281,15 @@ public class EntityStateSourcingListener { } } + private void onCalculatedFieldUpdate(Object entity, Object oldEntity) { + CalculatedField calculatedField = (CalculatedField) entity; + CalculatedField oldCalculatedField = null; + if (oldEntity instanceof CalculatedField) { + oldCalculatedField = (CalculatedField) oldEntity; + } + tbClusterService.onCalculatedFieldUpdated(calculatedField, oldCalculatedField, TbQueueCallback.EMPTY); + } + private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) { String data = JacksonUtil.toString(JacksonUtil.valueToTree(assignedDevice)); if (data != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java new file mode 100644 index 0000000000..4dfaec91cf --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -0,0 +1,106 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.entitiy.AbstractTbEntityService; +import org.thingsboard.server.service.security.model.SecurityUser; + +import java.util.Optional; + +@TbCoreComponent +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { + + private final CalculatedFieldService calculatedFieldService; + + @Override + public CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException { + ActionType actionType = calculatedField.getId() == null ? ActionType.ADDED : ActionType.UPDATED; + TenantId tenantId = calculatedField.getTenantId(); + try { + if (ActionType.UPDATED.equals(actionType)) { + CalculatedField existingCf = calculatedFieldService.findById(tenantId, calculatedField.getId()); + checkForEntityChange(existingCf, calculatedField); + } + checkEntityExistence(tenantId, calculatedField.getEntityId()); + CalculatedField savedCalculatedField = checkNotNull(calculatedFieldService.save(calculatedField)); + logEntityActionService.logEntityAction(tenantId, savedCalculatedField.getId(), savedCalculatedField, actionType, user); + return savedCalculatedField; + } catch (ThingsboardException e) { + logEntityActionService.logEntityAction(tenantId, emptyId(EntityType.CALCULATED_FIELD), calculatedField, actionType, user, e); + throw e; + } + } + + @Override + public CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user) { + return calculatedFieldService.findById(user.getTenantId(), calculatedFieldId); + } + + @Override + public PageData findAllByTenantIdAndEntityId(EntityId entityId, SecurityUser user, PageLink pageLink) { + TenantId tenantId = user.getTenantId(); + checkEntityExistence(tenantId, entityId); + return calculatedFieldService.findAllCalculatedFieldsByEntityId(tenantId, entityId, pageLink); + } + + @Override + @Transactional + public void delete(CalculatedField calculatedField, SecurityUser user) { + ActionType actionType = ActionType.DELETED; + TenantId tenantId = calculatedField.getTenantId(); + CalculatedFieldId calculatedFieldId = calculatedField.getId(); + try { + calculatedFieldService.deleteCalculatedField(tenantId, calculatedFieldId); + logEntityActionService.logEntityAction(tenantId, calculatedFieldId, calculatedField, actionType, user, calculatedFieldId.toString()); + } catch (Exception e) { + logEntityActionService.logEntityAction(tenantId, emptyId(EntityType.CALCULATED_FIELD), actionType, user, e, calculatedFieldId.toString()); + throw e; + } + } + + private void checkForEntityChange(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { + if (!oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId())) { + throw new IllegalArgumentException("Changing the calculated field target entity after initialization is prohibited."); + } + } + + private void checkEntityExistence(TenantId tenantId, EntityId entityId) { + switch (entityId.getEntityType()) { + case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> Optional.ofNullable(entityService.fetchEntity(tenantId, entityId)) + .orElseThrow(() -> new IllegalArgumentException(entityId.getEntityType().getNormalName() + " with id [" + entityId.getId() + "] does not exist.")); + default -> throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java new file mode 100644 index 0000000000..1e04a14a08 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.service.security.model.SecurityUser; + +public interface TbCalculatedFieldService { + + CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException; + + CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user); + + PageData findAllByTenantIdAndEntityId(EntityId entityId, SecurityUser user, PageLink pageLink); + + void delete(CalculatedField calculatedField, SecurityUser user); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java new file mode 100644 index 0000000000..bce7250bf7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.dao.cf.CalculatedFieldService; + +@Component +@RequiredArgsConstructor +@Slf4j +public class CalculatedFieldsDeletionTaskProcessor extends HousekeeperTaskProcessor { + + private final CalculatedFieldService calculatedFieldService; + + @Override + public void process(HousekeeperTask task) throws Exception { + int deletedCount = calculatedFieldService.deleteAllCalculatedFieldsByEntityId(task.getTenantId(), task.getEntityId()); + log.debug("[{}][{}][{}] Deleted {} calculated fields", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_CALCULATED_FIELDS; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index b2cde03931..870ce3838b 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -69,6 +69,7 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.mobile.app.MobileApp; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.page.PageLink; @@ -98,9 +99,9 @@ import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.mobile.MobileAppDao; import org.thingsboard.server.dao.notification.NotificationSettingsService; import org.thingsboard.server.dao.notification.NotificationTargetService; -import org.thingsboard.server.dao.mobile.MobileAppDao; import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.settings.AdminSettingsService; @@ -308,7 +309,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { jwtSettingsService.saveJwtSettings(jwtSettings); } - List mobiles = mobileAppDao.findByTenantId(TenantId.SYS_TENANT_ID, null, new PageLink(Integer.MAX_VALUE,0)).getData(); + List mobiles = mobileAppDao.findByTenantId(TenantId.SYS_TENANT_ID, null, new PageLink(Integer.MAX_VALUE, 0)).getData(); if (CollectionUtils.isNotEmpty(mobiles)) { mobiles.stream() .filter(mobileApp -> !validateKeyLength(mobileApp.getAppSecret())) @@ -571,7 +572,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { private void save(DeviceId deviceId, String key, boolean value) { if (persistActivityToTelemetry) { - ListenableFuture saveFuture = tsService.save( + ListenableFuture saveFuture = tsService.save( TenantId.SYS_TENANT_ID, deviceId, Collections.singletonList(new BasicTsKvEntry(System.currentTimeMillis(), new BooleanDataEntry(key, value))), 0L); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java new file mode 100644 index 0000000000..e76a2be9be --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -0,0 +1,269 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldLinkedTelemetryMsg; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.queue.QueueConfig; +import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldCache; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import org.thingsboard.server.service.queue.processing.AbstractConsumerService; +import org.thingsboard.server.service.queue.processing.IdMsgPair; +import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; + +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.thingsboard.server.common.util.ProtoUtils.fromProto; + +@Service +@TbRuleEngineComponent +@Slf4j +public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerService implements TbCalculatedFieldConsumerService { + + @Value("${queue.calculated_fields.poll_interval:25}") + private long pollInterval; + @Value("${queue.calculated_fields.pack_processing_timeout:60000}") + private long packProcessingTimeout; + @Value("${queue.calculated_fields.pool_size:8}") + private int poolSize; + + private final TbRuleEngineQueueFactory queueFactory; + private final CalculatedFieldStateService stateService; + + private PartitionedQueueConsumerManager> eventConsumer; + + public DefaultTbCalculatedFieldConsumerService(TbRuleEngineQueueFactory tbQueueFactory, + ActorSystemContext actorContext, + TbDeviceProfileCache deviceProfileCache, + TbAssetProfileCache assetProfileCache, + TbTenantProfileCache tenantProfileCache, + TbApiUsageStateService apiUsageStateService, + PartitionService partitionService, + ApplicationEventPublisher eventPublisher, + JwtSettingsService jwtSettingsService, + CalculatedFieldCache calculatedFieldCache, + CalculatedFieldStateService stateService) { + super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, + eventPublisher, jwtSettingsService); + this.queueFactory = tbQueueFactory; + this.stateService = stateService; + } + + @PostConstruct + public void init() { + super.init("tb-cf"); + + this.eventConsumer = PartitionedQueueConsumerManager.>create() + .queueKey(QueueKey.CF) + .topic(partitionService.getTopic(QueueKey.CF)) + .pollInterval(pollInterval) + .msgPackProcessor(this::processMsgs) + .consumerCreator((config, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer()) + .consumerExecutor(consumersExecutor) + .scheduler(scheduler) + .taskExecutor(mgmtExecutor) + .build(); + stateService.init(eventConsumer); + } + + @PreDestroy + public void destroy() { + super.destroy(); + } + + @Override + protected void startConsumers() { + super.startConsumers(); + } + + @Override + protected void onTbApplicationEvent(PartitionChangeEvent event) { + var partitions = event.getCfPartitions(); + try { + stateService.restore(partitions); + // eventConsumer's partitions will be updated by stateService + + // Cleanup old entities after corresponding consumers are stopped. + // Any periodic tasks need to check that the entity is still managed by the current server before processing. + actorContext.tell(new CalculatedFieldPartitionChangeMsg(partitionsToBooleanIndexArray(partitions))); + } catch (Throwable t) { + log.error("Failed to process partition change event: {}", event, t); + } + } + + private boolean[] partitionsToBooleanIndexArray(Set partitions) { + boolean[] myPartitions = new boolean[partitionService.getTotalCalculatedFieldPartitions()]; + for (var tpi : partitions) { + tpi.getPartition().ifPresent(partition -> myPartitions[partition] = true); + } + return myPartitions; + } + + private void processMsgs(List> msgs, TbQueueConsumer> consumer, QueueConfig config) throws Exception { + List> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList(); + ConcurrentMap> pendingMap = orderedMsgList.stream().collect( + Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + TbPackProcessingContext> ctx = new TbPackProcessingContext<>( + processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); + PendingMsgHolder pendingMsgHolder = new PendingMsgHolder<>(); + Future packSubmitFuture = consumersExecutor.submit(() -> { + orderedMsgList.forEach((element) -> { + UUID id = element.getUuid(); + TbProtoQueueMsg msg = element.getMsg(); + log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); + TbCallback callback = new TbPackCallback<>(id, ctx); + try { + ToCalculatedFieldMsg toCfMsg = msg.getValue(); + pendingMsgHolder.setMsg(toCfMsg); + if (toCfMsg.hasTelemetryMsg()) { + log.trace("[{}] Forwarding regular telemetry message for processing {}", id, toCfMsg.getTelemetryMsg()); + forwardToActorSystem(toCfMsg.getTelemetryMsg(), callback); + } else if (toCfMsg.hasLinkedTelemetryMsg()) { + forwardToActorSystem(toCfMsg.getLinkedTelemetryMsg(), callback); + } else if (toCfMsg.hasComponentLifecycleMsg()) { + log.trace("[{}] Forwarding component lifecycle message for processing {}", id, toCfMsg.getComponentLifecycleMsg()); + forwardToActorSystem(toCfMsg.getComponentLifecycleMsg(), callback); + } + } catch (Throwable e) { + log.warn("[{}] Failed to process message: {}", id, msg, e); + callback.onFailure(e); + } + }); + }); + if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + if (!packSubmitFuture.isDone()) { + packSubmitFuture.cancel(true); + log.info("Timeout to process message: {}", pendingMsgHolder.getMsg()); + } + if (log.isDebugEnabled()) { + ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); + } + ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); + } + consumer.commit(); + } + + @Override + protected ServiceType getServiceType() { + return ServiceType.TB_RULE_ENGINE; + } + + @Override + protected long getNotificationPollDuration() { + return pollInterval; + } + + @Override + protected long getNotificationPackProcessingTimeout() { + return packProcessingTimeout; + } + + @Override + protected int getMgmtThreadPoolSize() { + return Math.max(Runtime.getRuntime().availableProcessors(), 4); + } + + @Override + protected TbQueueConsumer> createNotificationsConsumer() { + return queueFactory.createToCalculatedFieldNotificationsMsgConsumer(); + } + + @Override + protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { + ToCalculatedFieldNotificationMsg toCfNotification = msg.getValue(); + if (toCfNotification.hasComponentLifecycleMsg()) { + // from upstream (maybe removed since we don't need to init state for each partition) + log.trace("[{}] Forwarding component lifecycle message for processing {}", id, toCfNotification.getComponentLifecycleMsg()); + forwardToActorSystem(toCfNotification.getComponentLifecycleMsg(), callback); + } else if (toCfNotification.hasLinkedTelemetryMsg()) { + forwardToActorSystem(toCfNotification.getLinkedTelemetryMsg(), callback); + } + } + + private void forwardToActorSystem(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { + var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); + var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + actorContext.tell(new CalculatedFieldTelemetryMsg(tenantId, entityId, msg, callback)); + } + + private void forwardToActorSystem(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { + var msg = linkedMsg.getMsg(); + var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); + var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + actorContext.tell(new CalculatedFieldLinkedTelemetryMsg(tenantId, entityId, linkedMsg, callback)); + } + + private void forwardToActorSystem(ComponentLifecycleMsgProto proto, TbCallback callback) { + var msg = fromProto(proto); + actorContext.tell(new CalculatedFieldEntityLifecycleMsg(msg.getTenantId(), msg, callback)); + } + + private TenantId toTenantId(long tenantIdMSB, long tenantIdLSB) { + return TenantId.fromUUID(new UUID(tenantIdMSB, tenantIdLSB)); + } + + @Override + protected void stopConsumers() { + super.stopConsumers(); + eventConsumer.stop(); + eventConsumer.awaitStop(); + stateService.stop(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index af298a9b87..800578ace4 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AssetId; @@ -77,6 +78,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.QueueDeleteMsg; import org.thingsboard.server.gen.transport.TransportProtos.QueueUpdateMsg; import org.thingsboard.server.gen.transport.TransportProtos.ResourceDeleteMsg; import org.thingsboard.server.gen.transport.TransportProtos.ResourceUpdateMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; @@ -91,8 +94,10 @@ import org.thingsboard.server.queue.common.MultipleTbQueueCallbackWrapper; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbRuleEngineProducerService; import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -141,6 +146,10 @@ public class DefaultTbClusterService implements TbClusterService { @Lazy private OtaPackageStateService otaPackageStateService; + @Autowired + @Lazy + private CalculatedFieldProcessingService calculatedFieldProcessingService; + private final TopicService topicService; private final TbDeviceProfileCache deviceProfileCache; private final TbAssetProfileCache assetProfileCache; @@ -182,6 +191,19 @@ public class DefaultTbClusterService implements TbClusterService { } } + @Override + public void broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg toCfMsg, TbQueueCallback callback) { + UUID msgId = UUID.randomUUID(); + TbQueueProducer> toCfProducer = producerProvider.getCalculatedFieldsNotificationsMsgProducer(); + Set tbReServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); + MultipleTbQueueCallbackWrapper callbackWrapper = new MultipleTbQueueCallbackWrapper(tbReServices.size(), callback); + for (String serviceId : tbReServices) { + TopicPartitionInfo tpi = topicService.getCalculatedFieldNotificationsTopic(serviceId); + toCfProducer.send(tpi, new TbProtoQueueMsg<>(msgId, toCfMsg), callbackWrapper); + toRuleEngineNfs.incrementAndGet(); + } + } + @Override public void pushMsgToVersionControl(TenantId tenantId, ToVersionControlServiceMsg msg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_VC_EXECUTOR, TenantId.SYS_TENANT_ID, tenantId); @@ -334,6 +356,26 @@ public class DefaultTbClusterService implements TbClusterService { toTransportNfs.incrementAndGet(); } + @Override + public void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF, entityId); + pushMsgToCalculatedFields(tpi, UUID.randomUUID(), msg, callback); + } + + @Override + public void pushMsgToCalculatedFields(TopicPartitionInfo tpi, UUID msgId, ToCalculatedFieldMsg msg, TbQueueCallback callback) { + log.trace("PUSHING msg: {} to:{}", msg, tpi); + producerProvider.getCalculatedFieldsMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); + toRuleEngineMsgs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS + } + + @Override + public void pushNotificationToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldNotificationMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF, entityId); + producerProvider.getCalculatedFieldsNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); + toRuleEngineNfs.incrementAndGet(); + } + @Override public void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); @@ -389,11 +431,19 @@ public class DefaultTbClusterService implements TbClusterService { public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) { DeviceId deviceId = device.getId(); gatewayNotificationsService.onDeviceDeleted(device); + handleCalculatedFieldEntityDeleted(tenantId, deviceId); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); } + @Override + public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { + AssetId assetId = asset.getId(); + handleCalculatedFieldEntityDeleted(tenantId, assetId); + broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); + } + @Override public void onDeviceAssignedToTenant(TenantId oldTenantId, Device device) { onDeviceDeleted(oldTenantId, device, null); @@ -553,7 +603,8 @@ public class DefaultTbClusterService implements TbClusterService { || entityType.equals(EntityType.API_USAGE_STATE) || (entityType.equals(EntityType.DEVICE) && msg.getEvent() == ComponentLifecycleEvent.UPDATED) || entityType.equals(EntityType.ENTITY_VIEW) - || entityType.equals(EntityType.NOTIFICATION_RULE)) { + || entityType.equals(EntityType.NOTIFICATION_RULE) + ) { TbQueueProducer> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); for (String serviceId : tbCoreServices) { @@ -604,21 +655,90 @@ public class DefaultTbClusterService implements TbClusterService { } @Override - public void onDeviceUpdated(Device device, Device old) { + public void onDeviceUpdated(Device entity, Device old) { var created = old == null; - broadcastEntityChangeToTransport(device.getTenantId(), device.getId(), device, null); + broadcastEntityChangeToTransport(entity.getTenantId(), entity.getId(), entity, null); if (old != null) { - boolean deviceNameChanged = !device.getName().equals(old.getName()); + boolean deviceNameChanged = !entity.getName().equals(old.getName()); if (deviceNameChanged) { - gatewayNotificationsService.onDeviceUpdated(device, old); + gatewayNotificationsService.onDeviceUpdated(entity, old); } - if (deviceNameChanged || !device.getType().equals(old.getType())) { - pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); + boolean deviceProfileChanged = !entity.getDeviceProfileId().equals(old.getDeviceProfileId()); + if (deviceProfileChanged) { + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(entity.getTenantId()) + .entityId(entity.getId()) + .event(ComponentLifecycleEvent.UPDATED) + .oldProfileId(old.getDeviceProfileId()) + .profileId(entity.getDeviceProfileId()) + .oldName(old.getName()) + .name(entity.getName()) + .build(); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + } + if (deviceNameChanged || deviceProfileChanged) { + pushMsgToCore(new DeviceNameOrTypeUpdateMsg(entity.getTenantId(), entity.getId(), entity.getName(), entity.getType()), null); } + } else { + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(entity.getTenantId()) + .entityId(entity.getId()) + .event(ComponentLifecycleEvent.CREATED) + .profileId(entity.getDeviceProfileId()) + .name(entity.getName()) + .build(); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } - broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), created, !created, false); - otaPackageStateService.update(device, old); + broadcastEntityStateChangeEvent(entity.getTenantId(), entity.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + sendDeviceStateServiceEvent(entity.getTenantId(), entity.getId(), created, !created, false); + otaPackageStateService.update(entity, old); + } + + @Override + public void onAssetUpdated(Asset entity, Asset old) { + var created = old == null; + if (old != null) { + boolean assetTypeChanged = !entity.getAssetProfileId().equals(old.getAssetProfileId()); + if (assetTypeChanged) { + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(entity.getTenantId()) + .entityId(entity.getId()) + .event(ComponentLifecycleEvent.UPDATED) + .oldProfileId(old.getAssetProfileId()) + .profileId(entity.getAssetProfileId()) + .oldName(old.getName()) + .name(entity.getName()) + .build(); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + } + } else { + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(entity.getTenantId()) + .entityId(entity.getId()) + .event(ComponentLifecycleEvent.CREATED) + .profileId(entity.getAssetProfileId()) + .name(entity.getName()) + .build(); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + } + broadcastEntityStateChangeEvent(entity.getTenantId(), entity.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + } + + @Override + public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, TbQueueCallback callback) { + var msg = toProto(new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), oldCalculatedField == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED)); + onCalculatedFieldLifecycleMsg(msg, callback); + } + + @Override + public void onCalculatedFieldDeleted(CalculatedField calculatedField, TbQueueCallback callback) { + var msg = toProto(new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), ComponentLifecycleEvent.DELETED)); + onCalculatedFieldLifecycleMsg(msg, callback); + } + + private void onCalculatedFieldLifecycleMsg(ComponentLifecycleMsgProto msg, TbQueueCallback callback) { + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(msg).build(), callback); + broadcastToCore(ToCoreNotificationMsg.newBuilder().setComponentLifecycle(msg).build()); } @Override @@ -748,4 +868,8 @@ public class DefaultTbClusterService implements TbClusterService { } } + private void handleCalculatedFieldEntityDeleted(TenantId tenantId, EntityId entityId) { + ComponentLifecycleMsg msg = new ComponentLifecycleMsg(tenantId, entityId, ComponentLifecycleEvent.DELETED); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index e156170770..de1bd7d367 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -20,8 +20,6 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; -import lombok.Getter; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; @@ -87,6 +85,7 @@ import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldCache; import org.thingsboard.server.service.notification.NotificationSchedulerService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -178,8 +177,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> ctx = new TbPackProcessingContext<>( processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); - PendingMsgHolder pendingMsgHolder = new PendingMsgHolder(); + PendingMsgHolder pendingMsgHolder = new PendingMsgHolder<>(); Future packSubmitFuture = consumersExecutor.submit(() -> { orderedMsgList.forEach((element) -> { UUID id = element.getUuid(); @@ -271,7 +271,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService(id, ctx); try { ToCoreMsg toCoreMsg = msg.getValue(); - pendingMsgHolder.setToCoreMsg(toCoreMsg); + pendingMsgHolder.setMsg(toCoreMsg); if (toCoreMsg.hasToSubscriptionMgrMsg()) { log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg()); forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback); @@ -322,8 +322,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); @@ -333,12 +332,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> ctx = new TbPackProcessingContext<>( processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); - PendingMsgHolder pendingMsgHolder = new PendingMsgHolder(); + PendingMsgHolder pendingMsgHolder = new PendingMsgHolder<>(); Future submitFuture = consumersExecutor.submit(() -> { orderedMsgList.forEach((element) -> { UUID id = element.getUuid(); @@ -145,7 +143,7 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService(id, ctx); try { ToEdgeMsg toEdgeMsg = msg.getValue(); - pendingMsgHolder.setToEdgeMsg(toEdgeMsg); + pendingMsgHolder.setMsg(toEdgeMsg); if (toEdgeMsg.hasEdgeNotificationMsg()) { pushNotificationToEdge(toEdgeMsg.getEdgeNotificationMsg(), 0, packProcessingRetries, callback); } @@ -161,20 +159,13 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService log.warn("[{}] Failed to process message: {}", id, msg.getValue())); } consumer.commit(); } - private static class PendingMsgHolder { - @Getter - @Setter - private volatile ToEdgeMsg toEdgeMsg; - } - @Override protected ServiceType getServiceType() { return ServiceType.TB_CORE; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index cabd126b1c..6391beecf8 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.rpc.RpcError; +import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -46,6 +47,7 @@ import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldCache; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; @@ -83,8 +85,9 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< TbApiUsageStateService apiUsageStateService, PartitionService partitionService, ApplicationEventPublisher eventPublisher, - JwtSettingsService jwtSettingsService) { - super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService); + JwtSettingsService jwtSettingsService, + CalculatedFieldCache calculatedFieldCache) { + super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService); this.ctx = ctx; this.tbDeviceRpcService = tbDeviceRpcService; this.queueService = queueService; @@ -105,6 +108,9 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { event.getNewPartitions().forEach((queueKey, partitions) -> { + if (CollectionsUtil.isOneOf(queueKey, QueueKey.CF, QueueKey.CF_STATES)) { + return; + } if (partitionService.isManagedByCurrentService(queueKey.getTenantId())) { var consumer = getConsumer(queueKey).orElseGet(() -> { Queue config = queueService.findQueueByTenantIdAndName(queueKey.getTenantId(), queueKey.getQueueName()); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/PendingMsgHolder.java b/application/src/main/java/org/thingsboard/server/service/queue/PendingMsgHolder.java new file mode 100644 index 0000000000..8f9cb3d092 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/PendingMsgHolder.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.Getter; +import lombok.Setter; + +public class PendingMsgHolder { + @Getter @Setter + private volatile T msg; +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java new file mode 100644 index 0000000000..8c7a459fab --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; + +public interface TbCalculatedFieldConsumerService extends ApplicationListener { + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java index 4c8edefd36..93112ca0f3 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.queue; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -23,9 +24,11 @@ import java.util.UUID; @Slf4j public class TbPackCallback implements TbCallback { private final TbPackProcessingContext ctx; + @Getter private final UUID id; public TbPackCallback(UUID id, TbPackProcessingContext ctx) { + log.trace("[{}] CALLBACK CREATED", id); this.id = id; this.ctx = ctx; } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 604f76aac1..d12bd896ff 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -25,6 +25,7 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -43,6 +44,7 @@ import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldCache; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.queue.TbPackCallback; @@ -68,6 +70,7 @@ public abstract class AbstractConsumerService findLwM2mObject(TenantId tenantId, String sortOrder, diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java index 3b5c257155..947095ab3a 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java @@ -19,6 +19,6 @@ public enum Operation { ALL, CREATE, READ, WRITE, DELETE, ASSIGN_TO_CUSTOMER, UNASSIGN_FROM_CUSTOMER, RPC_CALL, READ_CREDENTIALS, WRITE_CREDENTIALS, READ_ATTRIBUTES, WRITE_ATTRIBUTES, READ_TELEMETRY, WRITE_TELEMETRY, CLAIM_DEVICES, - ASSIGN_TO_TENANT + ASSIGN_TO_TENANT, READ_CALCULATED_FIELD, WRITE_CALCULATED_FIELD } diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java index edbc6c16b9..9d7590f786 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java @@ -50,7 +50,9 @@ public enum Resource { VERSION_CONTROL, NOTIFICATION(EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_RULE), - MOBILE_APP_SETTINGS; + MOBILE_APP_SETTINGS, + CALCULATED_FIELD(EntityType.CALCULATED_FIELD); + private final Set entityTypes; Resource() { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index 76c0e3cf62..a072cf2738 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.service.security.model.SecurityUser; -@Component(value="tenantAdminPermissions") +@Component(value = "tenantAdminPermissions") public class TenantAdminPermissions extends AbstractPermissions { public TenantAdminPermissions() { @@ -55,13 +55,13 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.OAUTH2_CONFIGURATION_TEMPLATE, new PermissionChecker.GenericPermissionChecker(Operation.READ)); put(Resource.MOBILE_APP, tenantEntityPermissionChecker); put(Resource.MOBILE_APP_BUNDLE, tenantEntityPermissionChecker); + put(Resource.CALCULATED_FIELD, tenantEntityPermissionChecker); } public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { @Override public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) { - if (!user.getTenantId().equals(entity.getTenantId())) { return false; } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java index a5c631b29c..7225156008 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java @@ -38,6 +38,7 @@ import org.thingsboard.server.service.subscription.SubscriptionManagerService; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Consumer; @@ -99,7 +100,11 @@ public abstract class AbstractSubscriptionService extends TbApplicationEventList } protected void addWsCallback(ListenableFuture saveFuture, Consumer callback) { - Futures.addCallback(saveFuture, new FutureCallback() { + addCallback(saveFuture, callback, wsCallBackExecutor); + } + + protected void addCallback(ListenableFuture saveFuture, Consumer callback, Executor executor) { + Futures.addCallback(saveFuture, new FutureCallback<>() { @Override public void onSuccess(@Nullable T result) { callback.accept(result); @@ -108,7 +113,15 @@ public abstract class AbstractSubscriptionService extends TbApplicationEventList @Override public void onFailure(Throwable t) { } - }, wsCallBackExecutor); + }, executor); + } + + protected static Consumer safeCallback(FutureCallback callback) { + if (callback != null) { + return callback::onFailure; + } else { + return throwable -> {}; + } } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 568d80787d..2bdbd917ab 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -41,6 +41,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -49,6 +50,7 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldQueueService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -75,6 +77,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final TbEntityViewService tbEntityViewService; private final TbApiUsageReportClient apiUsageClient; private final TbApiUsageStateService apiUsageStateService; + private final CalculatedFieldQueueService calculatedFieldQueueService; private ExecutorService tsCallBackExecutor; @@ -85,12 +88,14 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer TimeseriesService tsService, @Lazy TbEntityViewService tbEntityViewService, TbApiUsageReportClient apiUsageClient, - TbApiUsageStateService apiUsageStateService) { + TbApiUsageStateService apiUsageStateService, + CalculatedFieldQueueService calculatedFieldQueueService) { this.attrService = attrService; this.tsService = tsService; this.tbEntityViewService = tbEntityViewService; this.apiUsageClient = apiUsageClient; this.apiUsageStateService = apiUsageStateService; + this.calculatedFieldQueueService = calculatedFieldQueueService; } @PostConstruct @@ -120,10 +125,9 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer boolean sysTenant = TenantId.SYS_TENANT_ID.equals(tenantId) || tenantId == null; if (sysTenant || !request.getStrategy().saveTimeseries() || apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) { KvUtils.validate(request.getEntries(), valueNoXssValidation); - ListenableFuture future = saveTimeseriesInternal(request); + ListenableFuture future = saveTimeseriesInternal(request); if (request.getStrategy().saveTimeseries()) { - FutureCallback callback = getApiUsageCallback(tenantId, request.getCustomerId(), sysTenant, request.getCallback()); - Futures.addCallback(future, callback, tsCallBackExecutor); + Futures.addCallback(future, getApiUsageCallback(tenantId, request.getCustomerId(), sysTenant), tsCallBackExecutor); } } else { request.getCallback().onFailure(new RuntimeException("DB storage writes are disabled due to API limits!")); @@ -131,29 +135,36 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } @Override - public ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request) { + public ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); TimeseriesSaveRequest.Strategy strategy = request.getStrategy(); - ListenableFuture saveFuture; + ListenableFuture resultFuture; if (strategy.saveTimeseries() && strategy.saveLatest()) { - saveFuture = tsService.save(tenantId, entityId, request.getEntries(), request.getTtl()); + resultFuture = tsService.save(tenantId, entityId, request.getEntries(), request.getTtl()); } else if (strategy.saveLatest()) { - saveFuture = Futures.transform(tsService.saveLatest(tenantId, entityId, request.getEntries()), result -> 0, MoreExecutors.directExecutor()); + resultFuture = tsService.saveLatest(tenantId, entityId, request.getEntries()); } else if (strategy.saveTimeseries()) { - saveFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); + resultFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } else { - saveFuture = Futures.immediateFuture(0); + resultFuture = Futures.immediateFuture(TimeseriesSaveResult.EMPTY); } - addMainCallback(saveFuture, request.getCallback()); + addMainCallback(resultFuture, result -> { + if (strategy.processCalculatedFields()) { + calculatedFieldQueueService.pushRequestToQueue(request, result, request.getCallback()); + } else { + request.getCallback().onSuccess(null); + } + }, t -> request.getCallback().onFailure(t)); + if (strategy.sendWsUpdate()) { - addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); + addWsCallback(resultFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); } if (strategy.saveLatest()) { copyLatestToEntityViews(tenantId, entityId, request.getEntries()); } - return saveFuture; + return resultFuture; } @Override @@ -166,7 +177,9 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer public void saveAttributesInternal(AttributesSaveRequest request) { log.trace("Executing saveInternal [{}]", request); ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); - addMainCallback(saveFuture, request.getCallback()); + DonAsynchron.withCallback(saveFuture, result -> { + calculatedFieldQueueService.pushRequestToQueue(request, result, request.getCallback()); + }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); } @@ -179,7 +192,9 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void deleteAttributesInternal(AttributesDeleteRequest request) { ListenableFuture> deleteFuture = attrService.removeAll(request.getTenantId(), request.getEntityId(), request.getScope(), request.getKeys()); - addMainCallback(deleteFuture, request.getCallback()); + DonAsynchron.withCallback(deleteFuture, result -> { + calculatedFieldQueueService.pushRequestToQueue(request, result, request.getCallback()); + }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(deleteFuture, success -> onAttributesDelete(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getKeys(), request.isNotifyDevice())); } @@ -199,10 +214,14 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer deleteFuture = tsService.remove(request.getTenantId(), request.getEntityId(), request.getDeleteHistoryQueries()); addWsCallback(deleteFuture, result -> onTimeSeriesDelete(request.getTenantId(), request.getEntityId(), request.getKeys(), result)); } - addMainCallback(deleteFuture, __ -> request.getCallback().onSuccess(request.getKeys()), request.getCallback()::onFailure); + DonAsynchron.withCallback(deleteFuture, result -> { + calculatedFieldQueueService.pushRequestToQueue(request, request.getKeys(), getCalculatedFieldCallback(request.getCallback(), request.getKeys())); + }, safeCallback(getCalculatedFieldCallback(request.getCallback(), request.getKeys())), tsCallBackExecutor); } else { ListenableFuture> deleteFuture = tsService.removeAllLatest(request.getTenantId(), request.getEntityId()); - addMainCallback(deleteFuture, request.getCallback()::onSuccess, request.getCallback()::onFailure); + DonAsynchron.withCallback(deleteFuture, result -> { + calculatedFieldQueueService.pushRequestToQueue(request, request.getKeys(), getCalculatedFieldCallback(request.getCallback(), result)); + }, safeCallback(getCalculatedFieldCallback(request.getCallback(), request.getKeys())), tsCallBackExecutor); } } @@ -240,7 +259,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer .strategy(TimeseriesSaveRequest.Strategy.LATEST_AND_WS) .callback(new FutureCallback<>() { @Override - public void onSuccess(@Nullable Void tmp) {} + public void onSuccess(@Nullable Void tmp) { + } @Override public void onFailure(Throwable t) { @@ -262,27 +282,21 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } private void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice) { - forwardToSubscriptionManagerService(tenantId, entityId, subscriptionManagerService -> { - subscriptionManagerService.onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice, TbCallback.EMPTY); - }, () -> { - return TbSubscriptionUtils.toAttributesUpdateProto(tenantId, entityId, scope, attributes); - }); + forwardToSubscriptionManagerService(tenantId, entityId, + subscriptionManagerService -> subscriptionManagerService.onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice, TbCallback.EMPTY), + () -> TbSubscriptionUtils.toAttributesUpdateProto(tenantId, entityId, scope, attributes)); } private void onAttributesDelete(TenantId tenantId, EntityId entityId, String scope, List keys, boolean notifyDevice) { - forwardToSubscriptionManagerService(tenantId, entityId, subscriptionManagerService -> { - subscriptionManagerService.onAttributesDelete(tenantId, entityId, scope, keys, notifyDevice, TbCallback.EMPTY); - }, () -> { - return TbSubscriptionUtils.toAttributesDeleteProto(tenantId, entityId, scope, keys, notifyDevice); - }); + forwardToSubscriptionManagerService(tenantId, entityId, + subscriptionManagerService -> subscriptionManagerService.onAttributesDelete(tenantId, entityId, scope, keys, notifyDevice, TbCallback.EMPTY), + () -> TbSubscriptionUtils.toAttributesDeleteProto(tenantId, entityId, scope, keys, notifyDevice)); } private void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts) { - forwardToSubscriptionManagerService(tenantId, entityId, subscriptionManagerService -> { - subscriptionManagerService.onTimeSeriesUpdate(tenantId, entityId, ts, TbCallback.EMPTY); - }, () -> { - return TbSubscriptionUtils.toTimeseriesUpdateProto(tenantId, entityId, ts); - }); + forwardToSubscriptionManagerService(tenantId, entityId, + subscriptionManagerService -> subscriptionManagerService.onTimeSeriesUpdate(tenantId, entityId, ts, TbCallback.EMPTY), + () -> TbSubscriptionUtils.toTimeseriesUpdateProto(tenantId, entityId, ts)); } private void onTimeSeriesDelete(TenantId tenantId, EntityId entityId, List keys, List ts) { @@ -302,9 +316,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer subscriptionManagerService.onTimeSeriesUpdate(tenantId, entityId, updated, TbCallback.EMPTY); subscriptionManagerService.onTimeSeriesDelete(tenantId, entityId, deleted, TbCallback.EMPTY); - }, () -> { - return TbSubscriptionUtils.toTimeseriesDeleteProto(tenantId, entityId, keys); - }); + }, () -> TbSubscriptionUtils.toTimeseriesDeleteProto(tenantId, entityId, keys)); } private void addMainCallback(ListenableFuture saveFuture, final FutureCallback callback) { @@ -322,19 +334,32 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } } - private FutureCallback getApiUsageCallback(TenantId tenantId, CustomerId customerId, boolean sysTenant, FutureCallback callback) { + private FutureCallback getApiUsageCallback(TenantId tenantId, CustomerId customerId, boolean sysTenant) { return new FutureCallback<>() { @Override - public void onSuccess(Integer result) { - if (!sysTenant && result != null && result > 0) { - apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.STORAGE_DP_COUNT, result); + public void onSuccess(TimeseriesSaveResult result) { + Integer dataPoints = result.getDataPoints(); + if (!sysTenant && dataPoints != null && dataPoints > 0) { + apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.STORAGE_DP_COUNT, dataPoints); } - callback.onSuccess(null); } @Override public void onFailure(Throwable t) { - callback.onFailure(t); + } + }; + } + + private FutureCallback getCalculatedFieldCallback(FutureCallback> originalCallback, List keys) { + return new FutureCallback() { + @Override + public void onSuccess(Void unused) { + originalCallback.onSuccess(keys); + } + + @Override + public void onFailure(Throwable t) { + originalCallback.onFailure(t); } }; } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java index 380617934d..8a76aa1d14 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java @@ -21,13 +21,14 @@ import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; /** * Created by ashvayka on 27.03.18. */ public interface InternalTelemetryService extends RuleEngineTelemetryService { - ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request); + ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request); void saveAttributesInternal(AttributesSaveRequest request); diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java new file mode 100644 index 0000000000..77080c28c8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -0,0 +1,152 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils; + +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.util.KvProtoUtil; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsDoubleValProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsRollingArgumentProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; + +import java.util.Optional; +import java.util.TreeMap; +import java.util.UUID; + +public class CalculatedFieldUtils { + + public static CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { + return CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) + .build(); + } + + public static CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { + return CalculatedFieldEntityCtxIdProto.newBuilder() + .setTenantIdMSB(ctxId.tenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(ctxId.tenantId().getId().getLeastSignificantBits()) + .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) + .setEntityType(ctxId.entityId().getEntityType().name()) + .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) + .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) + .build(); + } + + public static CalculatedFieldEntityCtxId fromProto(CalculatedFieldEntityCtxIdProto ctxIdProto) { + TenantId tenantId = TenantId.fromUUID(new UUID(ctxIdProto.getTenantIdMSB(), ctxIdProto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); + return new CalculatedFieldEntityCtxId(tenantId, calculatedFieldId, entityId); + } + + public static CalculatedFieldStateProto toProto(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state) { + CalculatedFieldStateProto.Builder builder = CalculatedFieldStateProto.newBuilder() + .setId(toProto(stateId)) + .setType(state.getType().name()); + + state.getArguments().forEach((argName, argEntry) -> { + if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + builder.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); + } else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) { + builder.addRollingValueArguments(toRollingArgumentProto(argName, rollingArgumentEntry)); + } + }); + return builder.build(); + } + + public static SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { + SingleValueArgumentProto.Builder builder = SingleValueArgumentProto.newBuilder() + .setArgName(argName); + + if (entry.getKvEntryValue() != null) { + builder.setValue(KvProtoUtil.toTsValueProto(entry.getTs(), entry.getKvEntryValue())); + } + + Optional.ofNullable(entry.getVersion()).ifPresent(builder::setVersion); + + return builder.build(); + } + + public static TsRollingArgumentProto toRollingArgumentProto(String argName, TsRollingArgumentEntry entry) { + TsRollingArgumentProto.Builder builder = TsRollingArgumentProto.newBuilder() + .setKey(argName) + .setLimit(entry.getLimit()) + .setTimeWindow(entry.getTimeWindow()); + + entry.getTsRecords().forEach((ts, value) -> builder.addTsValue(TsDoubleValProto.newBuilder().setTs(ts).setValue(value).build())); + + return builder.build(); + } + + public static CalculatedFieldState fromProto(CalculatedFieldStateProto proto) { + if (StringUtils.isEmpty(proto.getType())) { + return null; + } + + CalculatedFieldType type = CalculatedFieldType.valueOf(proto.getType()); + + CalculatedFieldState state = switch (type) { + case SIMPLE -> new SimpleCalculatedFieldState(); + case SCRIPT -> new ScriptCalculatedFieldState(); + }; + + proto.getSingleValueArgumentsList().forEach(argProto -> + state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto))); + + if (CalculatedFieldType.SCRIPT.equals(type)) { + proto.getRollingValueArgumentsList().forEach(argProto -> + state.getArguments().put(argProto.getKey(), fromRollingArgumentProto(argProto))); + } + + return state; + } + + public static SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) { + if (!proto.hasValue()) { + return new SingleValueArgumentEntry(); + } + TsValueProto tsValueProto = proto.getValue(); + return new SingleValueArgumentEntry( + tsValueProto.getTs(), + (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto), + proto.getVersion() + ); + } + + public static TsRollingArgumentEntry fromRollingArgumentProto(TsRollingArgumentProto proto) { + TreeMap tsRecords = new TreeMap<>(); + proto.getTsValueList().forEach(tsValueProto -> tsRecords.put(tsValueProto.getTs(), tsValueProto.getValue())); + return new TsRollingArgumentEntry(tsRecords, proto.getLimit(), proto.getTimeWindow()); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/utils/DebugModeRateLimitsConfig.java b/application/src/main/java/org/thingsboard/server/utils/DebugModeRateLimitsConfig.java new file mode 100644 index 0000000000..99ef186a56 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/DebugModeRateLimitsConfig.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +@Data +public class DebugModeRateLimitsConfig { + + @Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.enabled:true}") + private boolean ruleChainDebugPerTenantLimitsEnabled; + @Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") + private String ruleChainDebugPerTenantLimitsConfiguration; + + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.enabled:true}") + private boolean calculatedFieldDebugPerTenantLimitsEnabled; + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") + private String calculatedFieldDebugPerTenantLimitsConfiguration; + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index f9045305ee..3db8b97e3a 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -437,6 +437,8 @@ actors: device_dispatcher_pool_size: "${ACTORS_SYSTEM_DEVICE_DISPATCHER_POOL_SIZE:4}" # Thread pool size for actor system dispatcher that process messages for device actors rule_dispatcher_pool_size: "${ACTORS_SYSTEM_RULE_DISPATCHER_POOL_SIZE:8}" # Thread pool size for actor system dispatcher that process messages for rule engine (chain/node) actors edge_dispatcher_pool_size: "${ACTORS_SYSTEM_EDGE_DISPATCHER_POOL_SIZE:4}" # Thread pool size for actor system dispatcher that process messages for edge actors + cfm_dispatcher_pool_size: "${ACTORS_SYSTEM_CFM_DISPATCHER_POOL_SIZE:2}" # Thread pool size for actor system dispatcher that process messages for CalculatedField manager actors + cfe_dispatcher_pool_size: "${ACTORS_SYSTEM_CFE_DISPATCHER_POOL_SIZE:8}" # Thread pool size for actor system dispatcher that process messages for CalculatedField entity actors tenant: create_components_on_init: "${ACTORS_TENANT_CREATE_COMPONENTS_ON_INIT:true}" # Create components in initialization session: @@ -504,6 +506,12 @@ actors: js_print_interval_ms: "${ACTORS_JS_STATISTICS_PRINT_INTERVAL_MS:10000}" # Actors statistic persistence frequency in milliseconds persist_frequency: "${ACTORS_STATISTICS_PERSIST_FREQUENCY:3600000}" + calculated_fields: + debug_mode_rate_limits_per_tenant: + # Enable/Disable the rate limit of persisted debug events for all calculated fields per tenant + enabled: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_ENABLED:true}" + # The value of DEBUG mode rate limit. By default, no more than 50 thousand events per hour + configuration: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" debug: settings: @@ -1279,8 +1287,9 @@ transport: # CoAP server parameters coap: - # Enable/disable coap server. - enabled: "${COAP_SERVER_ENABLED:true}" + server: + # Enable/disable coap server. + enabled: "${COAP_SERVER_ENABLED:true}" # CoAP bind address bind_address: "${COAP_BIND_ADDRESS:0.0.0.0}" # CoAP bind port @@ -1627,6 +1636,10 @@ queue: edge: "${TB_QUEUE_KAFKA_EDGE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Edge event topic edge-event: "${TB_QUEUE_KAFKA_EDGE_EVENT_TOPIC_PROPERTIES:retention.ms:2592000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Calculated Field topics + calculated-field: "${TB_QUEUE_KAFKA_CF_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Calculated Field State topics + calculated-field-state: "${TB_QUEUE_KAFKA_CF_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:104857600000;partitions:1;min.insync.replicas:1;cleanup.policy:compact}" # Kafka properties for EDQS events topics. Partitions number must be the same as queue.edqs.partitions edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1}" # Kafka properties for EDQS requests topic (default: 3 minutes retention). Partitions number must be the same as queue.edqs.partitions @@ -1787,6 +1800,21 @@ queue: topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:15}" # Size of the thread pool that handles such operations as partition changes, config updates, queue deletion management-thread-pool-size: "${TB_QUEUE_RULE_ENGINE_MGMT_THREAD_POOL_SIZE:12}" + calculated_fields: + # Topic name for Calculated Field (CF) events from Rule Engine + event_topic: "${TB_QUEUE_CF_EVENT_TOPIC:tb_cf_event}" + # Topic name for Calculated Field (CF) compacted states + state_topic: "${TB_QUEUE_CF_STATE_TOPIC:tb_cf_state}" + # Interval in milliseconds to poll messages by CF (Rule Engine) microservices + poll_interval: "${TB_QUEUE_CF_POLL_INTERVAL_MS:25}" + # Amount of partitions used by CF microservices + partitions: "${TB_QUEUE_CF_PARTITIONS:10}" + # Timeout for processing a message pack by CF microservices + pack_processing_timeout: "${TB_QUEUE_CF_PACK_PROCESSING_TIMEOUT_MS:60000}" + # Thread pool size for processing of the incoming messages + pool_size: "${TB_QUEUE_CF_POOL_SIZE:8}" + # RocksDB path for storing CF states + rocks_db_path: "${TB_QUEUE_CF_ROCKS_DB_PATH:${user.home}/.rocksdb/cf_states}" transport: # For high-priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" diff --git a/application/src/test/java/org/thingsboard/server/actors/rule/DefaultTbContextTest.java b/application/src/test/java/org/thingsboard/server/actors/rule/DefaultTbContextTest.java index af2f9ce8ec..54205327ca 100644 --- a/application/src/test/java/org/thingsboard/server/actors/rule/DefaultTbContextTest.java +++ b/application/src/test/java/org/thingsboard/server/actors/rule/DefaultTbContextTest.java @@ -98,6 +98,132 @@ class DefaultTbContextTest { defaultTbContext = new DefaultTbContext(mainCtxMock, "Test rule chain name", nodeCtxMock); } + @MethodSource + @ParameterizedTest + public void givenMsgWithQueueName_whenInput_thenVerifyEnqueueWithCorrectTpi(String queueName) { + // GIVEN + var tpi = resolve(queueName); + + given(mainCtxMock.resolve(eq(ServiceType.TB_RULE_ENGINE), eq(queueName), eq(TENANT_ID), eq(TENANT_ID))).willReturn(tpi); + var clusterService = mock(TbClusterService.class); + given(mainCtxMock.getClusterService()).willReturn(clusterService); + var callbackMock = mock(TbMsgCallback.class); + given(callbackMock.isMsgValid()).willReturn(true); + var ruleNode = new RuleNode(RULE_NODE_ID); + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(TENANT_ID) + .queueName(queueName) + .copyMetaData(TbMsgMetaData.EMPTY) + .data(TbMsg.EMPTY_STRING) + .callback(callbackMock) + .build(); + + var ruleChainId = new RuleChainId(UUID.randomUUID()); + + ruleNode.setRuleChainId(RULE_CHAIN_ID); + ruleNode.setDebugSettings(DebugSettings.failures()); + given(nodeCtxMock.getTenantId()).willReturn(TENANT_ID); + given(nodeCtxMock.getSelf()).willReturn(ruleNode); + + // WHEN + defaultTbContext.input(msg, ruleChainId); + + // THEN + then(clusterService).should().pushMsgToRuleEngine(eq(tpi), eq(msg.getId()), any(), any()); + } + + @MethodSource + @ParameterizedTest + public void givenMsgWithQueueName_whenEnqueue_thenVerifyEnqueueWithCorrectTpi(String queueName) { + // GIVEN + var tpi = resolve(queueName); + + given(mainCtxMock.resolve(eq(ServiceType.TB_RULE_ENGINE), eq(queueName), eq(TENANT_ID), eq(TENANT_ID))).willReturn(tpi); + var clusterService = mock(TbClusterService.class); + given(mainCtxMock.getClusterService()).willReturn(clusterService); + var callbackMock = mock(TbMsgCallback.class); + given(callbackMock.isMsgValid()).willReturn(true); + var ruleNode = new RuleNode(RULE_NODE_ID); + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(TENANT_ID) + .queueName(queueName) + .copyMetaData(TbMsgMetaData.EMPTY) + .data(TbMsg.EMPTY_STRING) + .callback(callbackMock) + .build(); + + ruleNode.setRuleChainId(RULE_CHAIN_ID); + ruleNode.setDebugSettings(DebugSettings.failures()); + given(nodeCtxMock.getTenantId()).willReturn(TENANT_ID); + + // WHEN + defaultTbContext.enqueue(msg, () -> {}, t -> {}); + + // THEN + then(clusterService).should().pushMsgToRuleEngine(eq(tpi), eq(msg.getId()), any(), any()); + } + + @MethodSource + @ParameterizedTest + public void givenMsgAndQueueName_whenEnqueue_thenVerifyEnqueueWithCorrectTpi(String queueName) { + // GIVEN + var tpi = resolve(queueName); + + given(mainCtxMock.resolve(eq(ServiceType.TB_RULE_ENGINE), eq(queueName), eq(TENANT_ID), eq(TENANT_ID))).willReturn(tpi); + var clusterService = mock(TbClusterService.class); + given(mainCtxMock.getClusterService()).willReturn(clusterService); + var callbackMock = mock(TbMsgCallback.class); + given(callbackMock.isMsgValid()).willReturn(true); + var ruleNode = new RuleNode(RULE_NODE_ID); + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(TENANT_ID) + .copyMetaData(TbMsgMetaData.EMPTY) + .data(TbMsg.EMPTY_STRING) + .callback(callbackMock) + .build(); + + ruleNode.setRuleChainId(RULE_CHAIN_ID); + ruleNode.setDebugSettings(DebugSettings.failures()); + given(nodeCtxMock.getTenantId()).willReturn(TENANT_ID); + + // WHEN + defaultTbContext.enqueue(msg, queueName, () -> {}, t -> {}); + + // THEN + then(clusterService).should().pushMsgToRuleEngine(eq(tpi), eq(msg.getId()), any(), any()); + } + + private static Stream givenMsgWithQueueName_whenInput_thenVerifyEnqueueWithCorrectTpi() { + return testQueueNames(); + } + + private static Stream givenMsgWithQueueName_whenEnqueue_thenVerifyEnqueueWithCorrectTpi() { + return testQueueNames(); + } + + private static Stream givenMsgAndQueueName_whenEnqueue_thenVerifyEnqueueWithCorrectTpi() { + return testQueueNames(); + } + + private static Stream testQueueNames() { + return Stream.of("Main", "Test", null); + } + + private TopicPartitionInfo resolve(String queueName) { + var tpiBuilder = TopicPartitionInfo.builder() + .topic(queueName == null ? "MainQueueTopic" : queueName + "QueueTopic") + .partition(1) + .myPartition(true); + + return tpiBuilder.build(); + } + @Test public void givenDebugFailuresEvents_whenTellSuccess_thenVerifyDebugOutputNotPersisted() { // GIVEN @@ -810,10 +936,10 @@ class DefaultTbContextTest { @MethodSource @ParameterizedTest void givenDebugFailuresAndDebugAllAndConnectionAndPersistedResultOptions_whenTellNext_thenVerifyDebugOutputPersistence(boolean debugFailures, - long debugAllUntil, - String connection, - boolean shouldPersist, - boolean shouldPersistAfterDurationTime) { + long debugAllUntil, + String connection, + boolean shouldPersist, + boolean shouldPersistAfterDurationTime) { // GIVEN var callbackMock = mock(TbMsgCallback.class); var msg = getTbMsgWithCallback(callbackMock); diff --git a/application/src/test/java/org/thingsboard/server/actors/tenant/TenantActorTest.java b/application/src/test/java/org/thingsboard/server/actors/tenant/TenantActorTest.java index 50349907fa..d8fac2b936 100644 --- a/application/src/test/java/org/thingsboard/server/actors/tenant/TenantActorTest.java +++ b/application/src/test/java/org/thingsboard/server/actors/tenant/TenantActorTest.java @@ -27,6 +27,7 @@ import org.thingsboard.server.actors.TbActorSystemSettings; import org.thingsboard.server.actors.TbEntityActorId; import org.thingsboard.server.actors.ruleChain.RuleChainActor; import org.thingsboard.server.actors.ruleChain.RuleChainToRuleChainMsg; +import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.RuleChainErrorActor; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.id.DeviceId; @@ -116,6 +117,7 @@ public class TenantActorTest { TbActorSystemSettings settings = new TbActorSystemSettings(0, 0, 0); TbActorSystem system = spy(new DefaultTbActorSystem(settings)); system.createDispatcher(RULE_DISPATCHER_NAME, mock()); + system.createDispatcher(DefaultActorService.CF_MANAGER_DISPATCHER_NAME, mock()); TbActorMailbox tenantCtx = new TbActorMailbox(system, settings, null, mock(), mock(), null); tenantActor.init(tenantCtx); diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java new file mode 100644 index 0000000000..9ddf0f0a04 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -0,0 +1,459 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cf; + +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.controller.CalculatedFieldControllerTest; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +@DaoSqlTest +public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTest { + + @BeforeEach + void setUp() throws Exception { + loginTenantAdmin(); + } + + @Test + public void testSimpleCalculatedFieldWhenAllTelemetryPresent() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":25}")); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"deviceTemperature\":40}")); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("C to F"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + argument.setDefaultValue("12"); // not used because real telemetry value in db is present + config.setArguments(Map.of("T", argument)); + config.setExpression("(T * 9/5) + 32"); + + Output output = new Output(); + output.setName("fahrenheitTemp"); + output.setType(OutputType.TIME_SERIES); + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + await().alias("create CF -> perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("77.0"); + }); + + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); + + await().alias("update telemetry -> recalculate state").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + }); + + Output savedOutput = savedCalculatedField.getConfiguration().getOutput(); + savedOutput.setType(OutputType.ATTRIBUTES); + savedOutput.setScope(AttributeScope.SERVER_SCOPE); + savedOutput.setName("temperatureF"); + savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); + + await().alias("update CF output -> perform calculation with updated output").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("86.0"); + }); + + Argument savedArgument = savedCalculatedField.getConfiguration().getArguments().get("T"); + savedArgument.setRefEntityKey(new ReferencedEntityKey("deviceTemperature", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); + savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); + + await().alias("update CF argument -> perform calculation with new argument").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); + }); + + savedCalculatedField.getConfiguration().setExpression("1.8 * T + 32"); + savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); + + await().alias("update CF expression -> perform calculation with new expression").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); + }); + } + + @Test + public void testSimpleCalculatedFieldWhenNotAllTelemetryPresent() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("C to F"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + config.setArguments(Map.of("T", argument)); + config.setExpression("(T * 9/5) + 32"); + + Output output = new Output(); + output.setName("fahrenheitTemp"); + output.setType(OutputType.TIME_SERIES); + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + await().alias("create CF -> state is not ready -> no calculation performed").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + }); + + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); + + await().alias("update telemetry -> perform calculation").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + }); + } + + @Test + public void testSimpleCalculatedFieldWhenNotAllTelemetryPresentButDefaultValueIsSet() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("C to F"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + argument.setDefaultValue("12"); + config.setArguments(Map.of("T", argument)); + config.setExpression("(T * 9/5) + 32"); + + Output output = new Output(); + output.setName("fahrenheitTemp"); + output.setType(OutputType.TIME_SERIES); + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + await().alias("create CF -> perform initial calculation with default value").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("53.6"); + }); + + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); + + await().alias("update telemetry -> recalculate state").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + }); + } + + @Test + public void testSimpleCalculatedFieldWhenEntityIdIsProfile() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":40}")); + + AssetProfile assetProfile = doPost("/api/assetProfile", createAssetProfile("Test Asset Profile"), AssetProfile.class); + + Asset asset1 = createAsset("Test asset 1", assetProfile.getId()); + doPost("/api/plugins/telemetry/ASSET/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":11}")); + + Asset asset2 = createAsset("Test asset 2", assetProfile.getId()); + doPost("/api/plugins/telemetry/ASSET/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":12}")); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(assetProfile.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("z = x + y"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument1 = new Argument(); + ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("y", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE); + argument1.setRefEntityKey(refEntityKey1); + + Argument argument2 = new Argument(); + argument2.setRefEntityId(testDevice.getId()); + ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("x", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE); + argument2.setRefEntityKey(refEntityKey2); + + config.setArguments(Map.of("x", argument2, "y", argument1)); + + config.setExpression("x + y"); + + Output output = new Output(); + output.setName("z"); + output.setType(OutputType.ATTRIBUTES); + output.setScope(AttributeScope.SERVER_SCOPE); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + await().alias("create CF and perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("51.0"); + + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("52.0"); + }); + + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":25}")); + + await().alias("update device telemetry -> recalculate state for all assets").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("36.0"); + + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + }); + + doPost("/api/plugins/telemetry/ASSET/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":15}")); + + await().alias("update asset 1 telemetry -> recalculate state only for asset 1").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); + + // result of asset 2 (no changes) + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + }); + + doPost("/api/plugins/telemetry/ASSET/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":5}")); + + await().alias("update asset 2 telemetry -> recalculate state only for asset 2").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 (no changes) + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); + + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("30.0"); + }); + + Asset asset3 = createAsset("Test asset 3", assetProfile.getId()); + doPost("/api/plugins/telemetry/ASSET/" + asset3.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":13}")); + + Asset finalAsset3 = asset3; + await().alias("add new entity to profile -> calculate state for new entity").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 3 + ArrayNode z3 = getServerAttributes(finalAsset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("38.0"); + }); + + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":20}")); + + await().alias("update device telemetry -> recalculate state for all assets").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("35.0"); + + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("25.0"); + + // result of asset 3 + ArrayNode z3 = getServerAttributes(finalAsset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + }); + + // update profile for asset 3 -> delete state for asset 3 + AssetProfile newAssetProfile = doPost("/api/assetProfile", createAssetProfile("New Asset Profile"), AssetProfile.class); + asset3.setAssetProfileId(newAssetProfile.getId()); + asset3 = doPost("/api/asset", asset3, Asset.class); + + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":15}")); + + Asset updatedAsset3 = asset3; + await().alias("update device telemetry -> recalculate state for asset 1 and asset 2").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("30.0"); + + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("20.0"); + + // no changes for asset 3 + ArrayNode z3 = getServerAttributes(updatedAsset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + }); + } + + @Test + public void testSimpleCalculatedFieldWhenExpressionIsInvalid() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":25}")); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("C to F"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + argument.setDefaultValue("12"); // not used because real telemetry value in db is present + config.setArguments(Map.of("T", argument)); + config.setExpression("(T * 9/0) + 32"); + + Output output = new Output(); + output.setName("fahrenheitTemp"); + output.setType(OutputType.TIME_SERIES); + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + await().alias("create CF -> ctx is not initialized -> no calculation perform").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + }); + + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); + + await().alias("update telemetry -> ctx is not initialized -> no calculation perform").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + }); + } + + private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { + return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class); + } + + private ArrayNode getServerAttributes(EntityId entityId, String... keys) throws Exception { + return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/attributes/SERVER_SCOPE?keys=" + String.join(",", keys), ArrayNode.class); + } + + private Asset createAsset(String name, AssetProfileId assetProfileId) { + Asset asset = new Asset(); + asset.setName(name); + asset.setAssetProfileId(assetProfileId); + return doPost("/api/asset", asset, Asset.class); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java index f1d7594746..d80c8a6ba5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java @@ -169,7 +169,7 @@ public class AlarmControllerTest extends AbstractControllerTest { foundAlarm = doGet("/api/alarm/info/" + updatedAlarm.getId(), AlarmInfo.class); testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(foundAlarm, customerDevice, tenantId, - customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_ACK, 1, 1, 1); + customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_ACK, 1, 0, 1); Mockito.reset(tbClusterService, auditLogService); alarm = updatedAlarm; @@ -183,7 +183,7 @@ public class AlarmControllerTest extends AbstractControllerTest { foundAlarm = doGet("/api/alarm/info/" + updatedAlarm.getId(), AlarmInfo.class); testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(foundAlarm, customerDevice, tenantId, - customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_CLEAR, 1, 1, 1); + customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_CLEAR, 1, 0, 1); Mockito.reset(tbClusterService, auditLogService); alarm = updatedAlarm; @@ -197,7 +197,7 @@ public class AlarmControllerTest extends AbstractControllerTest { foundAlarm = doGet("/api/alarm/info/" + updatedAlarm.getId(), AlarmInfo.class); testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(foundAlarm, customerDevice, tenantId, - customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_ASSIGNED, 1, 1, 1); + customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_ASSIGNED, 1, 0, 1); Mockito.reset(tbClusterService, auditLogService); alarm = updatedAlarm; @@ -211,7 +211,7 @@ public class AlarmControllerTest extends AbstractControllerTest { foundAlarm = doGet("/api/alarm/info/" + updatedAlarm.getId(), AlarmInfo.class); testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(foundAlarm, customerDevice, tenantId, - customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_UNASSIGNED, 1, 1, 1); + customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_UNASSIGNED, 1, 0, 1); Mockito.reset(tbClusterService, auditLogService); } @@ -321,7 +321,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertNotNull(foundAlarm); Assert.assertEquals(AlarmStatus.CLEARED_UNACK, foundAlarm.getStatus()); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ALARM_CLEAR); } @@ -337,7 +337,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertNotNull(foundAlarm); Assert.assertEquals(AlarmStatus.CLEARED_UNACK, foundAlarm.getStatus()); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_CLEAR); } @@ -354,7 +354,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertNotNull(foundAlarm); Assert.assertEquals(AlarmStatus.ACTIVE_ACK, foundAlarm.getStatus()); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ALARM_ACK); } @@ -434,7 +434,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertEquals(tenantAdminUserId, foundAlarm.getAssigneeId()); Assert.assertTrue(foundAlarm.getAssignTs() > beforeAssignmentTs && foundAlarm.getAssignTs() < System.currentTimeMillis()); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_ASSIGNED); } @@ -465,7 +465,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertEquals(tenantAdminUserId, foundAlarm.getAssigneeId()); Assert.assertTrue(foundAlarm.getAssignTs() > beforeAssignmentTs && foundAlarm.getAssignTs() < System.currentTimeMillis()); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_ASSIGNED); logout(); @@ -482,7 +482,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertEquals(customerUserId, foundAlarm.getAssigneeId()); Assert.assertTrue(foundAlarm.getAssignTs() > beforeAssignmentTs && foundAlarm.getAssignTs() < System.currentTimeMillis()); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ALARM_ASSIGNED); } @@ -500,7 +500,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertEquals(tenantAdminUserId, foundAlarm.getAssigneeId()); Assert.assertTrue(foundAlarm.getAssignTs() > beforeAssignmentTs && foundAlarm.getAssignTs() < System.currentTimeMillis()); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_ASSIGNED); beforeAssignmentTs = System.currentTimeMillis(); @@ -511,7 +511,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertNull(foundAlarm.getAssigneeId()); Assert.assertTrue(foundAlarm.getAssignTs() > beforeAssignmentTs && foundAlarm.getAssignTs() < System.currentTimeMillis()); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_UNASSIGNED); } @@ -529,7 +529,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertEquals(tenantAdminUserId, foundAlarm.getAssigneeId()); Assert.assertTrue(foundAlarm.getAssignTs() > beforeAssignmentTs && foundAlarm.getAssignTs() < System.currentTimeMillis()); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ALARM_ASSIGNED); logout(); @@ -545,7 +545,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertNull(foundAlarm.getAssigneeId()); Assert.assertTrue(foundAlarm.getAssignTs() > beforeAssignmentTs && foundAlarm.getAssignTs() < System.currentTimeMillis()); - testNotifyEntityAllOneTime(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundAlarm, foundAlarm.getId(), foundAlarm.getOriginator(), tenantId, customerId, customerUserId, CUSTOMER_USER_EMAIL, ActionType.ALARM_UNASSIGNED); } diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java new file mode 100644 index 0000000000..ee66f664cc --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -0,0 +1,163 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@DaoSqlTest +public class CalculatedFieldControllerTest extends AbstractControllerTest { + + private Tenant savedTenant; + + @Before + public void beforeTest() throws Exception { + loginSysAdmin(); + + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + savedTenant = saveTenant(tenant); + assertThat(savedTenant).isNotNull(); + + User tenantAdmin = new User(); + tenantAdmin.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin.setTenantId(savedTenant.getId()); + tenantAdmin.setEmail("tenant2@thingsboard.org"); + tenantAdmin.setFirstName("Joe"); + tenantAdmin.setLastName("Downs"); + + createUserAndLogin(tenantAdmin, "testPassword1"); + } + + @After + public void afterTest() throws Exception { + loginSysAdmin(); + + deleteTenant(savedTenant.getId()); + } + + @Test + public void testSaveCalculatedField() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + CalculatedField calculatedField = getCalculatedField(testDevice.getId()); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + assertThat(savedCalculatedField).isNotNull(); + assertThat(savedCalculatedField.getId()).isNotNull(); + assertThat(savedCalculatedField.getCreatedTime()).isGreaterThan(0); + assertThat(savedCalculatedField.getTenantId()).isEqualTo(savedTenant.getId()); + assertThat(savedCalculatedField.getEntityId()).isEqualTo(calculatedField.getEntityId()); + assertThat(savedCalculatedField.getType()).isEqualTo(calculatedField.getType()); + assertThat(savedCalculatedField.getName()).isEqualTo(calculatedField.getName()); + assertThat(savedCalculatedField.getConfiguration()).isEqualTo(getCalculatedFieldConfig(testDevice.getId())); + assertThat(savedCalculatedField.getVersion()).isEqualTo(1L); + + savedCalculatedField.setName("Test CF"); + + CalculatedField updatedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); + + assertThat(updatedCalculatedField.getName()).isEqualTo(savedCalculatedField.getName()); + assertThat(updatedCalculatedField.getVersion()).isEqualTo(savedCalculatedField.getVersion() + 1); + + doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) + .andExpect(status().isOk()); + } + + @Test + public void testGetCalculatedFieldById() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + CalculatedField calculatedField = getCalculatedField(testDevice.getId()); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + CalculatedField fetchedCalculatedField = doGet("/api/calculatedField/" + savedCalculatedField.getId().getId(), CalculatedField.class); + + assertThat(fetchedCalculatedField).isNotNull(); + assertThat(fetchedCalculatedField).isEqualTo(savedCalculatedField); + + doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) + .andExpect(status().isOk()); + } + + @Test + public void testDeleteCalculatedField() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + CalculatedField calculatedField = getCalculatedField(testDevice.getId()); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + assertThat(savedCalculatedField).isNotNull(); + + doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) + .andExpect(status().isOk()); + doGet("/api/calculatedField/" + savedCalculatedField.getId().getId()).andExpect(status().isNotFound()); + } + + private CalculatedField getCalculatedField(DeviceId deviceId) { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(deviceId); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(getCalculatedFieldConfig(null)); + calculatedField.setVersion(1L); + return calculatedField; + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + argument.setRefEntityId(referencedEntityId); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + + config.setArguments(Map.of("T", argument)); + + config.setExpression("T - (100 - H) / 5"); + + Output output = new Output(); + output.setName("output"); + output.setType(OutputType.TIME_SERIES); + + config.setOutput(output); + + return config; + } + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java index 9688ce8758..320e3815f6 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java @@ -27,6 +27,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Primary; import org.springframework.test.context.ContextConfiguration; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.Device; @@ -60,6 +61,7 @@ import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; +import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.containsString; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.common.data.DataConstants.DEFAULT_DEVICE_TYPE; @@ -79,6 +81,8 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { @Autowired private DeviceProfileDao deviceProfileDao; + static final String LWM2M_PROFILE_JSON = "{\"name\":\"lwm2m profile\",\"type\":\"DEFAULT\",\"image\":null,\"defaultQueueName\":null,\"transportType\":\"LWM2M\",\"provisionType\":\"DISABLED\",\"description\":\"\",\"profileData\":{\"configuration\":{\"type\":\"DEFAULT\"},\"transportConfiguration\":{\"observeAttr\":{\"observe\":[],\"attribute\":[],\"telemetry\":[\"/11_1.1/0/0\"],\"keyName\":{\"/11_1.1/0/0\":\"profileName\"},\"attributeLwm2m\":{}},\"bootstrap\":[{\"shortServerId\":123,\"bootstrapServerIs\":false,\"host\":\"0.0.0.0\",\"port\":5685,\"clientHoldOffTime\":1,\"serverPublicKey\":\"\",\"serverCertificate\":\"\",\"bootstrapServerAccountTimeout\":0,\"lifetime\":300,\"defaultMinPeriod\":1,\"notifIfDisabled\":true,\"binding\":\"U\",\"securityMode\":\"NO_SEC\"}],\"clientLwM2mSettings\":{\"clientOnlyObserveAfterConnect\":1,\"fwUpdateStrategy\":1,\"swUpdateStrategy\":1,\"powerMode\":\"DRX\",\"edrxCycle\":81000,\"psmActivityTimer\":10000,\"pagingTransmissionWindow\":10000,\"defaultObjectIDVer\":\"1.0\"},\"bootstrapServerUpdateEnable\":false,\"type\":\"LWM2M\"},\"alarms\":null,\"provisionConfiguration\":{\"type\":\"DISABLED\"}}}"; + static class Config { @Bean @Primary @@ -119,7 +123,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { Mockito.reset(tbClusterService, auditLogService); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile savedDeviceProfile = saveDeviceProfile(deviceProfile); Assert.assertNotNull(savedDeviceProfile); Assert.assertNotNull(savedDeviceProfile.getId()); Assert.assertTrue(savedDeviceProfile.getCreatedTime() > 0); @@ -135,7 +139,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { ActionType.ADDED); savedDeviceProfile.setName("New device profile"); - doPost("/api/deviceProfile", savedDeviceProfile, DeviceProfile.class); + saveDeviceProfile(savedDeviceProfile); DeviceProfile foundDeviceProfile = doGet("/api/deviceProfile/" + savedDeviceProfile.getId().getId().toString(), DeviceProfile.class); Assert.assertEquals(savedDeviceProfile.getName(), foundDeviceProfile.getName()); @@ -162,7 +166,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { @Test public void testFindDeviceProfileById() throws Exception { DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile savedDeviceProfile = saveDeviceProfile(deviceProfile); DeviceProfile foundDeviceProfile = doGet("/api/deviceProfile/" + savedDeviceProfile.getId().getId().toString(), DeviceProfile.class); Assert.assertNotNull(foundDeviceProfile); Assert.assertEquals(savedDeviceProfile, foundDeviceProfile); @@ -171,7 +175,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { @Test public void whenGetDeviceProfileById_thenPermissionsAreChecked() throws Exception { DeviceProfile deviceProfile = createDeviceProfile("Device profile 1", null); - deviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + deviceProfile = saveDeviceProfile(deviceProfile); loginDifferentTenant(); @@ -183,7 +187,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { @Test public void testFindDeviceProfileInfoById() throws Exception { DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile savedDeviceProfile = saveDeviceProfile(deviceProfile); DeviceProfileInfo foundDeviceProfileInfo = doGet("/api/deviceProfileInfo/" + savedDeviceProfile.getId().getId().toString(), DeviceProfileInfo.class); Assert.assertNotNull(foundDeviceProfileInfo); Assert.assertEquals(savedDeviceProfile.getId(), foundDeviceProfileInfo.getId()); @@ -213,7 +217,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { @Test public void whenGetDeviceProfileInfoById_thenPermissionsAreChecked() throws Exception { DeviceProfile deviceProfile = createDeviceProfile("Device profile 1", null); - deviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + deviceProfile = saveDeviceProfile(deviceProfile); loginDifferentTenant(); doGet("/api/deviceProfileInfo/" + deviceProfile.getId()) @@ -235,7 +239,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { @Test public void testSetDefaultDeviceProfile() throws Exception { DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile 1"); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile savedDeviceProfile = saveDeviceProfile(deviceProfile); Mockito.reset(tbClusterService, auditLogService); @@ -328,7 +332,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { @Test public void testChangeDeviceProfileTypeNull() throws Exception { DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile savedDeviceProfile = saveDeviceProfile(deviceProfile); Mockito.reset(tbClusterService, auditLogService); @@ -345,7 +349,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { @Test public void testChangeDeviceProfileTransportTypeWithExistingDevices() throws Exception { DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile savedDeviceProfile = saveDeviceProfile(deviceProfile); Device device = new Device(); device.setName("Test device"); device.setType("default"); @@ -367,7 +371,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { @Test public void testDeleteDeviceProfileWithExistingDevice() throws Exception { DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile savedDeviceProfile = saveDeviceProfile(deviceProfile); Device device = new Device(); device.setName("Test device"); @@ -419,7 +423,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { public void testSaveDeviceProfileWithFirmwareFromDifferentTenant() throws Exception { loginDifferentTenant(); DeviceProfile differentProfile = createDeviceProfile("Different profile"); - differentProfile = doPost("/api/deviceProfile", differentProfile, DeviceProfile.class); + differentProfile = saveDeviceProfile(differentProfile); SaveOtaPackageInfoRequest firmwareInfo = new SaveOtaPackageInfoRequest(); firmwareInfo.setDeviceProfileId(differentProfile.getId()); firmwareInfo.setType(FIRMWARE); @@ -441,7 +445,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { public void testSaveDeviceProfileWithSoftwareFromDifferentTenant() throws Exception { loginDifferentTenant(); DeviceProfile differentProfile = createDeviceProfile("Different profile"); - differentProfile = doPost("/api/deviceProfile", differentProfile, DeviceProfile.class); + differentProfile = saveDeviceProfile(differentProfile); SaveOtaPackageInfoRequest softwareInfo = new SaveOtaPackageInfoRequest(); softwareInfo.setDeviceProfileId(differentProfile.getId()); softwareInfo.setType(SOFTWARE); @@ -462,7 +466,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { @Test public void testDeleteDeviceProfile() throws Exception { DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile savedDeviceProfile = saveDeviceProfile(deviceProfile); Mockito.reset(tbClusterService, auditLogService); @@ -495,7 +499,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { int cntEntity = 28; for (int i = 0; i < cntEntity; i++) { DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile" + i); - deviceProfiles.add(doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class)); + deviceProfiles.add(saveDeviceProfile(deviceProfile)); } testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(new DeviceProfile(), new DeviceProfile(), @@ -552,7 +556,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { for (int i = 0; i < 28; i++) { DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile" + i); - deviceProfiles.add(doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class)); + deviceProfiles.add(saveDeviceProfile(deviceProfile)); } List loadedDeviceProfileInfos = new ArrayList<>(); @@ -961,7 +965,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { JsonTransportPayloadConfiguration jsonTransportPayloadConfiguration = new JsonTransportPayloadConfiguration(); MqttDeviceProfileTransportConfiguration mqttDeviceProfileTransportConfiguration = this.createMqttDeviceProfileTransportConfiguration(jsonTransportPayloadConfiguration, true); DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile", mqttDeviceProfileTransportConfiguration); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile savedDeviceProfile = saveDeviceProfile(deviceProfile); Assert.assertNotNull(savedDeviceProfile); Assert.assertEquals(savedDeviceProfile.getTransportType(), DeviceTransportType.MQTT); Assert.assertTrue(savedDeviceProfile.getProfileData().getTransportConfiguration() instanceof MqttDeviceProfileTransportConfiguration); @@ -979,7 +983,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { "v1/devices/me/telemetry", "v1/devices/me/attributes", "v1/devices/me/subscribeattributes"); DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile", mqttDeviceProfileTransportConfiguration); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile savedDeviceProfile = saveDeviceProfile(deviceProfile); Assert.assertNotNull(savedDeviceProfile); Assert.assertEquals(savedDeviceProfile.getTransportType(), DeviceTransportType.MQTT); Assert.assertTrue(savedDeviceProfile.getProfileData().getTransportConfiguration() instanceof MqttDeviceProfileTransportConfiguration); @@ -997,7 +1001,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = this.createProtoTransportPayloadConfiguration(schema, schema, null, null); MqttDeviceProfileTransportConfiguration mqttDeviceProfileTransportConfiguration = this.createMqttDeviceProfileTransportConfiguration(protoTransportPayloadConfiguration, false); DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile", mqttDeviceProfileTransportConfiguration); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile savedDeviceProfile = saveDeviceProfile(deviceProfile); Assert.assertNotNull(savedDeviceProfile); DeviceProfile foundDeviceProfile = doGet("/api/deviceProfile/" + savedDeviceProfile.getId().getId().toString(), DeviceProfile.class); Assert.assertEquals(savedDeviceProfile, foundDeviceProfile); @@ -1036,14 +1040,14 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { @Test public void testDeleteDeviceProfileWithDeleteRelationsOk() throws Exception { - DeviceProfileId deviceProfileId = savedDeviceProfile("DeviceProfile for Test WithRelationsOk").getId(); + DeviceProfileId deviceProfileId = saveDeviceProfile("DeviceProfile for Test WithRelationsOk").getId(); testEntityDaoWithRelationsOk(savedTenant.getId(), deviceProfileId, "/api/deviceProfile/" + deviceProfileId); } @Ignore @Test public void testDeleteDeviceProfileExceptionWithRelationsTransactional() throws Exception { - DeviceProfileId deviceProfileId = savedDeviceProfile("DeviceProfile for Test WithRelations Transactional Exception").getId(); + DeviceProfileId deviceProfileId = saveDeviceProfile("DeviceProfile for Test WithRelations Transactional Exception").getId(); testEntityDaoWithRelationsTransactionalException(deviceProfileDao, savedTenant.getId(), deviceProfileId, "/api/deviceProfile/" + deviceProfileId); } @@ -1103,8 +1107,36 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { Assert.assertEquals(count, deviceProfileNames.size()); } - private DeviceProfile savedDeviceProfile(String name) { + @Test + public void testSaveDeviceProfileWithOutdatedVersion() throws Exception { + DeviceProfile deviceProfile = JacksonUtil.fromString(LWM2M_PROFILE_JSON, DeviceProfile.class); + deviceProfile.setName("Device profile v1.0"); + deviceProfile = saveDeviceProfile(deviceProfile); + assertThat(deviceProfile.getVersion()).isOne(); + + deviceProfile.setName("Device profile v2.0"); + deviceProfile = saveDeviceProfile(deviceProfile); + assertThat(deviceProfile.getVersion()).isEqualTo(2); + + deviceProfile.setName("Device profile v1.1"); + deviceProfile.setVersion(1L); + String response = doPost("/api/deviceProfile", deviceProfile).andExpect(status().isConflict()) + .andReturn().getResponse().getContentAsString(); + assertThat(JacksonUtil.toJsonNode(response).get("message").asText()) + .containsIgnoringCase("already changed by someone else"); + + deviceProfile.setVersion(null); // overriding entity + deviceProfile = saveDeviceProfile(deviceProfile); + assertThat(deviceProfile.getName()).isEqualTo("Device profile v1.1"); + assertThat(deviceProfile.getVersion()).isEqualTo(3); + } + + private DeviceProfile saveDeviceProfile(String name) { DeviceProfile deviceProfile = createDeviceProfile(name); + return saveDeviceProfile(deviceProfile); + } + + private DeviceProfile saveDeviceProfile(DeviceProfile deviceProfile) { return doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); } diff --git a/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java index c4074f0d00..0d7b81370a 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.io.IOUtils; import org.junit.After; import org.junit.Assert; @@ -27,7 +28,10 @@ import org.springframework.http.HttpHeaders; import org.springframework.mock.web.MockHttpServletResponse; import org.springframework.test.web.servlet.ResultActions; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.DashboardInfo; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TbResource; @@ -40,13 +44,16 @@ import org.thingsboard.server.common.data.lwm2m.LwM2mObject; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; +import org.thingsboard.server.common.data.widget.WidgetTypeInfo; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DaoSqlTest; import java.util.ArrayList; import java.util.Base64; import java.util.Collections; +import java.util.HashMap; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; @@ -63,7 +70,6 @@ public class TbResourceControllerTest extends AbstractControllerTest { private static final String JS_TEST_FILE_NAME = "test.js"; private static final String TEST_DATA = "77u/PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPCEtLQpGSUxFIElORk9STUFUSU9OCgpPTUEgUGVybWFuZW50IERvY3VtZW50CiAgIEZpbGU6IE9NQS1TVVAtTHdNMk1fQmluYXJ5QXBwRGF0YUNvbnRhaW5lci1WMV8wXzEtMjAxOTAyMjEtQQogICBUeXBlOiB4bWwKClB1YmxpYyBSZWFjaGFibGUgSW5mb3JtYXRpb24KICAgUGF0aDogaHR0cDovL3d3dy5vcGVubW9iaWxlYWxsaWFuY2Uub3JnL3RlY2gvcHJvZmlsZXMKICAgTmFtZTogTHdNMk1fQmluYXJ5QXBwRGF0YUNvbnRhaW5lci12MV8wXzEueG1sCgpOT1JNQVRJVkUgSU5GT1JNQVRJT04KCiAgSW5mb3JtYXRpb24gYWJvdXQgdGhpcyBmaWxlIGNhbiBiZSBmb3VuZCBpbiB0aGUgbGF0ZXN0IHJldmlzaW9uIG9mCgogIE9NQS1UUy1MV00yTV9CaW5hcnlBcHBEYXRhQ29udGFpbmVyLVYxXzBfMQoKICBUaGlzIGlzIGF2YWlsYWJsZSBhdCBodHRwOi8vd3d3Lm9wZW5tb2JpbGVhbGxpYW5jZS5vcmcvCgogIFNlbmQgY29tbWVudHMgdG8gaHR0cHM6Ly9naXRodWIuY29tL09wZW5Nb2JpbGVBbGxpYW5jZS9PTUFfTHdNMk1fZm9yX0RldmVsb3BlcnMvaXNzdWVzCgpDSEFOR0UgSElTVE9SWQoKMTUwNjIwMTggU3RhdHVzIGNoYW5nZWQgdG8gQXBwcm92ZWQgYnkgRE0sIERvYyBSZWYgIyBPTUEtRE0mU0UtMjAxOC0wMDYxLUlOUF9MV00yTV9BUFBEQVRBX1YxXzBfRVJQX2Zvcl9maW5hbF9BcHByb3ZhbAoyMTAyMjAxOSBTdGF0dXMgY2hhbmdlZCB0byBBcHByb3ZlZCBieSBJUFNPLCBEb2MgUmVmICMgT01BLUlQU08tMjAxOS0wMDI1LUlOUF9Md00yTV9PYmplY3RfQXBwX0RhdGFfQ29udGFpbmVyXzFfMF8xX2Zvcl9GaW5hbF9BcHByb3ZhbAoKTEVHQUwgRElTQ0xBSU1FUgoKQ29weXJpZ2h0IDIwMTkgT3BlbiBNb2JpbGUgQWxsaWFuY2UuCgpSZWRpc3RyaWJ1dGlvbiBhbmQgdXNlIGluIHNvdXJjZSBhbmQgYmluYXJ5IGZvcm1zLCB3aXRoIG9yIHdpdGhvdXQKbW9kaWZpY2F0aW9uLCBhcmUgcGVybWl0dGVkIHByb3ZpZGVkIHRoYXQgdGhlIGZvbGxvd2luZyBjb25kaXRpb25zCmFyZSBtZXQ6CgoxLiBSZWRpc3RyaWJ1dGlvbnMgb2Ygc291cmNlIGNvZGUgbXVzdCByZXRhaW4gdGhlIGFib3ZlIGNvcHlyaWdodApub3RpY2UsIHRoaXMgbGlzdCBvZiBjb25kaXRpb25zIGFuZCB0aGUgZm9sbG93aW5nIGRpc2NsYWltZXIuCjIuIFJlZGlzdHJpYnV0aW9ucyBpbiBiaW5hcnkgZm9ybSBtdXN0IHJlcHJvZHVjZSB0aGUgYWJvdmUgY29weXJpZ2h0Cm5vdGljZSwgdGhpcyBsaXN0IG9mIGNvbmRpdGlvbnMgYW5kIHRoZSBmb2xsb3dpbmcgZGlzY2xhaW1lciBpbiB0aGUKZG9jdW1lbnRhdGlvbiBhbmQvb3Igb3RoZXIgbWF0ZXJpYWxzIHByb3ZpZGVkIHdpdGggdGhlIGRpc3RyaWJ1dGlvbi4KMy4gTmVpdGhlciB0aGUgbmFtZSBvZiB0aGUgY29weXJpZ2h0IGhvbGRlciBub3IgdGhlIG5hbWVzIG9mIGl0cwpjb250cmlidXRvcnMgbWF5IGJlIHVzZWQgdG8gZW5kb3JzZSBvciBwcm9tb3RlIHByb2R1Y3RzIGRlcml2ZWQKZnJvbSB0aGlzIHNvZnR3YXJlIHdpdGhvdXQgc3BlY2lmaWMgcHJpb3Igd3JpdHRlbiBwZXJtaXNzaW9uLgoKVEhJUyBTT0ZUV0FSRSBJUyBQUk9WSURFRCBCWSBUSEUgQ09QWVJJR0hUIEhPTERFUlMgQU5EIENPTlRSSUJVVE9SUwoiQVMgSVMiIEFORCBBTlkgRVhQUkVTUyBPUiBJTVBMSUVEIFdBUlJBTlRJRVMsIElOQ0xVRElORywgQlVUIE5PVApMSU1JVEVEIFRPLCBUSEUgSU1QTElFRCBXQVJSQU5USUVTIE9GIE1FUkNIQU5UQUJJTElUWSBBTkQgRklUTkVTUwpGT1IgQSBQQVJUSUNVTEFSIFBVUlBPU0UgQVJFIERJU0NMQUlNRUQuIElOIE5PIEVWRU5UIFNIQUxMIFRIRQpDT1BZUklHSFQgSE9MREVSIE9SIENPTlRSSUJVVE9SUyBCRSBMSUFCTEUgRk9SIEFOWSBESVJFQ1QsIElORElSRUNULApJTkNJREVOVEFMLCBTUEVDSUFMLCBFWEVNUExBUlksIE9SIENPTlNFUVVFTlRJQUwgREFNQUdFUyAoSU5DTFVESU5HLApCVVQgTk9UIExJTUlURUQgVE8sIFBST0NVUkVNRU5UIE9GIFNVQlNUSVRVVEUgR09PRFMgT1IgU0VSVklDRVM7CkxPU1MgT0YgVVNFLCBEQVRBLCBPUiBQUk9GSVRTOyBPUiBCVVNJTkVTUyBJTlRFUlJVUFRJT04pIEhPV0VWRVIKQ0FVU0VEIEFORCBPTiBBTlkgVEhFT1JZIE9GIExJQUJJTElUWSwgV0hFVEhFUiBJTiBDT05UUkFDVCwgU1RSSUNUCkxJQUJJTElUWSwgT1IgVE9SVCAoSU5DTFVESU5HIE5FR0xJR0VOQ0UgT1IgT1RIRVJXSVNFKSBBUklTSU5HIElOCkFOWSBXQVkgT1VUIE9GIFRIRSBVU0UgT0YgVEhJUyBTT0ZUV0FSRSwgRVZFTiBJRiBBRFZJU0VEIE9GIFRIRQpQT1NTSUJJTElUWSBPRiBTVUNIIERBTUFHRS4KClRoZSBhYm92ZSBsaWNlbnNlIGlzIHVzZWQgYXMgYSBsaWNlbnNlIHVuZGVyIGNvcHlyaWdodCBvbmx5LiBQbGVhc2UKcmVmZXJlbmNlIHRoZSBPTUEgSVBSIFBvbGljeSBmb3IgcGF0ZW50IGxpY2Vuc2luZyB0ZXJtczoKaHR0cHM6Ly93d3cub21hc3BlY3dvcmtzLm9yZy9hYm91dC9pbnRlbGxlY3R1YWwtcHJvcGVydHktcmlnaHRzLwoKLS0+CjxMV00yTSB4bWxuczp4c2k9Imh0dHA6Ly93d3cudzMub3JnLzIwMDEvWE1MU2NoZW1hLWluc3RhbmNlIiB4c2k6bm9OYW1lc3BhY2VTY2hlbWFMb2NhdGlvbj0iaHR0cDovL29wZW5tb2JpbGVhbGxpYW5jZS5vcmcvdGVjaC9wcm9maWxlcy9MV00yTS54c2QiPgoJPE9iamVjdCBPYmplY3RUeXBlPSJNT0RlZmluaXRpb24iPgoJCTxOYW1lPkJpbmFyeUFwcERhdGFDb250YWluZXI8L05hbWU+CgkJPERlc2NyaXB0aW9uMT48IVtDREFUQVtUaGlzIEx3TTJNIE9iamVjdHMgcHJvdmlkZXMgdGhlIGFwcGxpY2F0aW9uIHNlcnZpY2UgZGF0YSByZWxhdGVkIHRvIGEgTHdNMk0gU2VydmVyLCBlZy4gV2F0ZXIgbWV0ZXIgZGF0YS4gClRoZXJlIGFyZSBzZXZlcmFsIG1ldGhvZHMgdG8gY3JlYXRlIGluc3RhbmNlIHRvIGluZGljYXRlIHRoZSBtZXNzYWdlIGRpcmVjdGlvbiBiYXNlZCBvbiB0aGUgbmVnb3RpYXRpb24gYmV0d2VlbiBBcHBsaWNhdGlvbiBhbmQgTHdNMk0uIFRoZSBDbGllbnQgYW5kIFNlcnZlciBzaG91bGQgbmVnb3RpYXRlIHRoZSBpbnN0YW5jZShzKSB1c2VkIHRvIGV4Y2hhbmdlIHRoZSBkYXRhLiBGb3IgZXhhbXBsZToKIC0gVXNpbmcgYSBzaW5nbGUgaW5zdGFuY2UgZm9yIGJvdGggZGlyZWN0aW9ucyBjb21tdW5pY2F0aW9uLCBmcm9tIENsaWVudCB0byBTZXJ2ZXIgYW5kIGZyb20gU2VydmVyIHRvIENsaWVudC4KIC0gVXNpbmcgYW4gaW5zdGFuY2UgZm9yIGNvbW11bmljYXRpb24gZnJvbSBDbGllbnQgdG8gU2VydmVyIGFuZCBhbm90aGVyIG9uZSBmb3IgY29tbXVuaWNhdGlvbiBmcm9tIFNlcnZlciB0byBDbGllbnQKIC0gVXNpbmcgc2V2ZXJhbCBpbnN0YW5jZXMKXV0+PC9EZXNjcmlwdGlvbjE+CgkJPE9iamVjdElEPjE5PC9PYmplY3RJRD4KCQk8T2JqZWN0VVJOPnVybjpvbWE6bHdtMm06b21hOjE5PC9PYmplY3RVUk4+CgkJPExXTTJNVmVyc2lvbj4xLjA8L0xXTTJNVmVyc2lvbj4KCQk8T2JqZWN0VmVyc2lvbj4xLjA8L09iamVjdFZlcnNpb24+CgkJPE11bHRpcGxlSW5zdGFuY2VzPk11bHRpcGxlPC9NdWx0aXBsZUluc3RhbmNlcz4KCQk8TWFuZGF0b3J5Pk9wdGlvbmFsPC9NYW5kYXRvcnk+CgkJPFJlc291cmNlcz4KCQkJPEl0ZW0gSUQ9IjAiPjxOYW1lPkRhdGE8L05hbWU+CgkJCQk8T3BlcmF0aW9ucz5SVzwvT3BlcmF0aW9ucz4KCQkJCTxNdWx0aXBsZUluc3RhbmNlcz5NdWx0aXBsZTwvTXVsdGlwbGVJbnN0YW5jZXM+CgkJCQk8TWFuZGF0b3J5Pk1hbmRhdG9yeTwvTWFuZGF0b3J5PgoJCQkJPFR5cGU+T3BhcXVlPC9UeXBlPgoJCQkJPFJhbmdlRW51bWVyYXRpb24gLz4KCQkJCTxVbml0cyAvPgoJCQkJPERlc2NyaXB0aW9uPjwhW0NEQVRBW0luZGljYXRlcyB0aGUgYXBwbGljYXRpb24gZGF0YSBjb250ZW50Ll1dPjwvRGVzY3JpcHRpb24+CgkJCTwvSXRlbT4KCQkJPEl0ZW0gSUQ9IjEiPjxOYW1lPkRhdGEgUHJpb3JpdHk8L05hbWU+CgkJCQk8T3BlcmF0aW9ucz5SVzwvT3BlcmF0aW9ucz4KCQkJCTxNdWx0aXBsZUluc3RhbmNlcz5TaW5nbGU8L011bHRpcGxlSW5zdGFuY2VzPgoJCQkJPE1hbmRhdG9yeT5PcHRpb25hbDwvTWFuZGF0b3J5PgoJCQkJPFR5cGU+SW50ZWdlcjwvVHlwZT4KCQkJCTxSYW5nZUVudW1lcmF0aW9uPjEgYnl0ZXM8L1JhbmdlRW51bWVyYXRpb24+CgkJCQk8VW5pdHMgLz4KCQkJCTxEZXNjcmlwdGlvbj48IVtDREFUQVtJbmRpY2F0ZXMgdGhlIEFwcGxpY2F0aW9uIGRhdGEgcHJpb3JpdHk6CjA6SW1tZWRpYXRlCjE6QmVzdEVmZm9ydAoyOkxhdGVzdAozLTEwMDogUmVzZXJ2ZWQgZm9yIGZ1dHVyZSB1c2UuCjEwMS0yNTQ6IFByb3ByaWV0YXJ5IG1vZGUuXV0+PC9EZXNjcmlwdGlvbj4KCQkJPC9JdGVtPgoJCQk8SXRlbSBJRD0iMiI+PE5hbWU+RGF0YSBDcmVhdGlvbiBUaW1lPC9OYW1lPgoJCQkJPE9wZXJhdGlvbnM+Ulc8L09wZXJhdGlvbnM+CgkJCQk8TXVsdGlwbGVJbnN0YW5jZXM+U2luZ2xlPC9NdWx0aXBsZUluc3RhbmNlcz4KCQkJCTxNYW5kYXRvcnk+T3B0aW9uYWw8L01hbmRhdG9yeT4KCQkJCTxUeXBlPlRpbWU8L1R5cGU+CgkJCQk8UmFuZ2VFbnVtZXJhdGlvbiAvPgoJCQkJPFVuaXRzIC8+CgkJCQk8RGVzY3JpcHRpb24+PCFbQ0RBVEFbSW5kaWNhdGVzIHRoZSBEYXRhIGluc3RhbmNlIGNyZWF0aW9uIHRpbWVzdGFtcC5dXT48L0Rlc2NyaXB0aW9uPgoJCQk8L0l0ZW0+CgkJCTxJdGVtIElEPSIzIj48TmFtZT5EYXRhIERlc2NyaXB0aW9uPC9OYW1lPgoJCQkJPE9wZXJhdGlvbnM+Ulc8L09wZXJhdGlvbnM+CgkJCQk8TXVsdGlwbGVJbnN0YW5jZXM+U2luZ2xlPC9NdWx0aXBsZUluc3RhbmNlcz4KCQkJCTxNYW5kYXRvcnk+T3B0aW9uYWw8L01hbmRhdG9yeT4KCQkJCTxUeXBlPlN0cmluZzwvVHlwZT4KCQkJCTxSYW5nZUVudW1lcmF0aW9uPjMyIGJ5dGVzPC9SYW5nZUVudW1lcmF0aW9uPgoJCQkJPFVuaXRzIC8+CgkJCQk8RGVzY3JpcHRpb24+PCFbQ0RBVEFbSW5kaWNhdGVzIHRoZSBkYXRhIGRlc2NyaXB0aW9uLgplLmcuICJtZXRlciByZWFkaW5nIi5dXT48L0Rlc2NyaXB0aW9uPgoJCQk8L0l0ZW0+CgkJCTxJdGVtIElEPSI0Ij48TmFtZT5EYXRhIEZvcm1hdDwvTmFtZT4KCQkJCTxPcGVyYXRpb25zPlJXPC9PcGVyYXRpb25zPgoJCQkJPE11bHRpcGxlSW5zdGFuY2VzPlNpbmdsZTwvTXVsdGlwbGVJbnN0YW5jZXM+CgkJCQk8TWFuZGF0b3J5Pk9wdGlvbmFsPC9NYW5kYXRvcnk+CgkJCQk8VHlwZT5TdHJpbmc8L1R5cGU+CgkJCQk8UmFuZ2VFbnVtZXJhdGlvbj4zMiBieXRlczwvUmFuZ2VFbnVtZXJhdGlvbj4KCQkJCTxVbml0cyAvPgoJCQkJPERlc2NyaXB0aW9uPjwhW0NEQVRBW0luZGljYXRlcyB0aGUgZm9ybWF0IG9mIHRoZSBBcHBsaWNhdGlvbiBEYXRhLgplLmcuIFlHLU1ldGVyLVdhdGVyLVJlYWRpbmcKVVRGOC1zdHJpbmcKXV0+PC9EZXNjcmlwdGlvbj4KCQkJPC9JdGVtPgoJCQk8SXRlbSBJRD0iNSI+PE5hbWU+QXBwIElEPC9OYW1lPgoJCQkJPE9wZXJhdGlvbnM+Ulc8L09wZXJhdGlvbnM+CgkJCQk8TXVsdGlwbGVJbnN0YW5jZXM+U2luZ2xlPC9NdWx0aXBsZUluc3RhbmNlcz4KCQkJCTxNYW5kYXRvcnk+T3B0aW9uYWw8L01hbmRhdG9yeT4KCQkJCTxUeXBlPkludGVnZXI8L1R5cGU+CgkJCQk8UmFuZ2VFbnVtZXJhdGlvbj4yIGJ5dGVzPC9SYW5nZUVudW1lcmF0aW9uPgoJCQkJPFVuaXRzIC8+CgkJCQk8RGVzY3JpcHRpb24+PCFbQ0RBVEFbSW5kaWNhdGVzIHRoZSBkZXN0aW5hdGlvbiBBcHBsaWNhdGlvbiBJRC5dXT48L0Rlc2NyaXB0aW9uPgoJCQk8L0l0ZW0+PC9SZXNvdXJjZXM+CgkJPERlc2NyaXB0aW9uMj48IVtDREFUQVtdXT48L0Rlc2NyaXB0aW9uMj4KCTwvT2JqZWN0Pgo8L0xXTTJNPgo="; - private Tenant savedTenant; private User tenantAdmin; @@ -222,28 +228,196 @@ public class TbResourceControllerTest extends AbstractControllerTest { } @Test - public void testShoudNotDeleteTbResourceIfAssignedToWidgetType() throws Exception { + public void testUnForcedDeleteTbResourceIfAssignedToWidgetType() throws Exception { TbResource resource = new TbResource(); - resource.setResourceType(ResourceType.JKS); + resource.setResourceType(ResourceType.JS_MODULE); resource.setTitle("My first resource"); - resource.setFileName(DEFAULT_FILE_NAME); + resource.setFileName(JS_TEST_FILE_NAME); + resource.setTenantId(savedTenant.getId()); resource.setEncodedData(TEST_DATA); + resource.setResourceKey(JS_TEST_FILE_NAME); TbResourceInfo savedResource = save(resource); - Mockito.reset(tbClusterService, auditLogService); - String resourceIdStr = savedResource.getId().getId().toString(); + var link = resource.getLink(); + WidgetTypeDetails widgetType = new WidgetTypeDetails(); + widgetType.setName("Widget Type"); + widgetType.setTenantId(savedTenant.getId()); + widgetType.setDescriptor(JacksonUtil.newObjectNode() + .put("controllerScript", "self.onInit = function() {\n self.ctx.$scope.actionWidget.onInit();\n}\n\nself.typeParameters = function() {\n return {\n previewWidth: '300px',\n previewHeight: '320px',\n embedTitlePanel: true,\n targetDeviceOptional: true,\n displayRpcMessageToast: false\n };\n};\n\nself.onDestroy = function() {\n}") + .put("settingsSchema", "") + .put("dataKeySettingsSchema", "{}\n") + .put("settingsDirective", "tb-scada-symbol-widget-settings") + .put("hasBasicMode", true) + .put("basicModeDirective", "tb-scada-symbol-basic-config") + .put("resource", link)); + WidgetType savedWidgetType = doPost("/api/widgetType", widgetType, WidgetTypeDetails.class); + + var deleteResponse = doDelete("/api/resource/" + savedResource.getUuidId() + "?force=false") + .andExpect(status().isBadRequest()) + .andReturn() + .getResponse() + .getContentAsString(); + + Assert.assertNotNull(deleteResponse); + + boolean isSuccess = JacksonUtil.toJsonNode(deleteResponse).get("success").asBoolean(); + Assert.assertFalse(isSuccess); + + var referenceValues = JacksonUtil.toJsonNode(deleteResponse).get("references"); + Assert.assertNotNull(referenceValues); + + var widgetTypeInfos = JacksonUtil.readValue(referenceValues.toString(), new TypeReference>>() { + }); + Assert.assertNotNull(widgetTypeInfos); + Assert.assertFalse(widgetTypeInfos.isEmpty()); + Assert.assertEquals(1, widgetTypeInfos.size()); + + var dashboardInfo = widgetTypeInfos.get(EntityType.WIDGET_TYPE.name()).get(0); + Assert.assertNotNull(dashboardInfo); - //create widget type + WidgetTypeInfo foundedWidgetType = doGet("/api/widgetTypeInfo/" + savedWidgetType.getId().getId().toString(), WidgetTypeInfo.class); + Assert.assertNotNull(foundedWidgetType); + Assert.assertEquals(foundedWidgetType, dashboardInfo); + } + + @Test + public void testForcedDeleteTbResourceIfAssignedToWidgetType() throws Exception { + TbResource resource = new TbResource(); + resource.setResourceType(ResourceType.JS_MODULE); + resource.setTitle("My first resource"); + resource.setFileName(JS_TEST_FILE_NAME); + resource.setTenantId(savedTenant.getId()); + resource.setEncodedData(TEST_DATA); + resource.setResourceKey(JS_TEST_FILE_NAME); + TbResourceInfo savedResource = save(resource); + + var link = resource.getLink(); WidgetTypeDetails widgetType = new WidgetTypeDetails(); widgetType.setName("Widget Type"); - widgetType.setDescriptor(JacksonUtil.fromString(String.format("{ \"resources\": [{\"url\":\"tb-resource;/api/resource/jks/tenant/%s\",\"isModule\":true}]}", savedResource.getResourceKey()), JsonNode.class)); + widgetType.setTenantId(savedTenant.getId()); + widgetType.setDescriptor(JacksonUtil.newObjectNode() + .put("controllerScript", "self.onInit = function() {\n self.ctx.$scope.actionWidget.onInit();\n}\n\nself.typeParameters = function() {\n return {\n previewWidth: '300px',\n previewHeight: '320px',\n embedTitlePanel: true,\n targetDeviceOptional: true,\n displayRpcMessageToast: false\n };\n};\n\nself.onDestroy = function() {\n}") + .put("settingsSchema", "") + .put("dataKeySettingsSchema", "{}\n") + .put("settingsDirective", "tb-scada-symbol-widget-settings") + .put("hasBasicMode", true) + .put("basicModeDirective", "tb-scada-symbol-basic-config") + .put("resource", link)); doPost("/api/widgetType", widgetType, WidgetTypeDetails.class); - doDelete("/api/resource/" + resourceIdStr) + var deleteResponse = doDelete("/api/resource/" + savedResource.getUuidId() + "?force=true") + .andExpect(status().isOk()) + .andReturn() + .getResponse() + .getContentAsString(); + + Assert.assertNotNull(deleteResponse); + + boolean isSuccess = JacksonUtil.toJsonNode(deleteResponse).get("success").asBoolean(); + Assert.assertTrue(isSuccess); + + var referenceValues = JacksonUtil.toJsonNode(deleteResponse).get("references"); + var widgetTypeInfos = JacksonUtil.readValue(referenceValues.toString(), new TypeReference>>() { + }); + Assert.assertNull(widgetTypeInfos); + } + + @Test + public void testUnForcedDeleteTbResourceIfAssignedToDashboard() throws Exception { + TbResource resource = new TbResource(); + resource.setResourceType(ResourceType.JS_MODULE); + resource.setTitle("My first resource"); + resource.setFileName(JS_TEST_FILE_NAME); + resource.setTenantId(savedTenant.getId()); + resource.setEncodedData(TEST_DATA); + resource.setResourceKey(JS_TEST_FILE_NAME); + TbResourceInfo savedResource = save(resource); + + var link = resource.getLink(); + Dashboard dashboard = new Dashboard(); + dashboard.setTitle("My dashboard"); + dashboard.setTenantId(savedTenant.getId()); + dashboard.setConfiguration(JacksonUtil.newObjectNode() + .set("widgets", JacksonUtil.toJsonNode(""" + {"xxx": + {"config":{"actions":{"elementClick":[ + {"customResources":[{"url":{"entityType":"TB_RESOURCE","id": + "tb-resource;/api/resource/js_module/tenant/gateway-management-extension.js"},"isModule":true}, + {"url":"tb-resource;/api/resource/js_module/tenant/gateway-management-extension.js","isModule":true}]}]}}}} + """)) + .put("resource", link)); + Dashboard savedDashboard = doPost("/api/dashboard", dashboard, Dashboard.class); + + var deleteResponse = doDelete("/api/resource/" + savedResource.getUuidId() + "?force=false") .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Following widget types use this resource: " - + widgetType.getName()))); + .andReturn() + .getResponse() + .getContentAsString(); + + Assert.assertNotNull(deleteResponse); + + boolean isSuccess = JacksonUtil.toJsonNode(deleteResponse).get("success").asBoolean(); + Assert.assertFalse(isSuccess); + + var referenceValues = JacksonUtil.toJsonNode(deleteResponse).get("references"); + Assert.assertNotNull(referenceValues); + + var dashboardInfos = JacksonUtil.readValue(referenceValues.toString(), new TypeReference>>() { + }); + Assert.assertNotNull(dashboardInfos); + Assert.assertFalse(dashboardInfos.isEmpty()); + Assert.assertEquals(1, dashboardInfos.size()); + + var dashboardInfo = dashboardInfos.get(EntityType.DASHBOARD.name()).get(0); + Assert.assertNotNull(dashboardInfo); + + DashboardInfo foundDashboard = doGet("/api/dashboard/info/" + savedDashboard.getId().getId().toString(), DashboardInfo.class); + Assert.assertNotNull(foundDashboard); + Assert.assertEquals(foundDashboard, dashboardInfo); + } + + @Test + public void testForcedDeleteTbResourceIfAssignedToDashboard() throws Exception { + TbResource resource = new TbResource(); + resource.setResourceType(ResourceType.JS_MODULE); + resource.setTitle("My first resource"); + resource.setFileName(JS_TEST_FILE_NAME); + resource.setTenantId(savedTenant.getId()); + resource.setEncodedData(TEST_DATA); + resource.setResourceKey(JS_TEST_FILE_NAME); + TbResourceInfo savedResource = save(resource); + + var link = resource.getLink(); + Dashboard dashboard = new Dashboard(); + dashboard.setTitle("My dashboard"); + dashboard.setTenantId(savedTenant.getId()); + dashboard.setConfiguration(JacksonUtil.newObjectNode() + .set("widgets", JacksonUtil.toJsonNode(""" + {"xxx": + {"config":{"actions":{"elementClick":[ + {"customResources":[{"url":{"entityType":"TB_RESOURCE","id": + "tb-resource;/api/resource/js_module/tenant/gateway-management-extension.js"},"isModule":true}, + {"url":"tb-resource;/api/resource/js_module/tenant/gateway-management-extension.js","isModule":true}]}]}}}} + """)) + .put("resource", link)); + doPost("/api/dashboard", dashboard, Dashboard.class); + + var deleteResponse = doDelete("/api/resource/" + savedResource.getUuidId() + "?force=true") + .andExpect(status().isOk()) + .andReturn() + .getResponse() + .getContentAsString(); + + Assert.assertNotNull(deleteResponse); + + boolean isSuccess = JacksonUtil.toJsonNode(deleteResponse).get("success").asBoolean(); + Assert.assertTrue(isSuccess); + + var referenceValues = JacksonUtil.toJsonNode(deleteResponse).get("references"); + var dashboardInfos = JacksonUtil.readValue(referenceValues.toString(), new TypeReference>>() { + }); + Assert.assertNull(dashboardInfos); } @Test @@ -676,7 +850,8 @@ public class TbResourceControllerTest extends AbstractControllerTest { List resources = loadLwm2mResources(); List objects = - doGetTyped("/api/resource/lwm2m/page?pageSize=100&page=0", new TypeReference<>() {}); + doGetTyped("/api/resource/lwm2m/page?pageSize=100&page=0", new TypeReference<>() { + }); Assert.assertNotNull(objects); Assert.assertEquals(resources.size(), objects.size()); @@ -690,7 +865,8 @@ public class TbResourceControllerTest extends AbstractControllerTest { List resources = loadLwm2mResources(); List objects = - doGetTyped("/api/resource/lwm2m?sortProperty=id&sortOrder=ASC&objectIds=3_1.2,5_1.2,19_1.1", new TypeReference<>() {}); + doGetTyped("/api/resource/lwm2m?sortProperty=id&sortOrder=ASC&objectIds=3_1.2,5_1.2,19_1.1", new TypeReference<>() { + }); Assert.assertNotNull(objects); Assert.assertEquals(3, objects.size()); diff --git a/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java index 6d953c1722..0daa56728b 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.queue.TbQueueCallback; import java.util.ArrayList; import java.util.Collections; @@ -44,6 +45,7 @@ import java.util.List; import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -354,7 +356,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { argument -> argument.getClass().equals(TenantProfile.class); if (ComponentLifecycleEvent.DELETED.equals(event)) { Mockito.verify(tbClusterService, times(cntTime)).onTenantProfileDelete(Mockito.argThat(matcherTenantProfile), - Mockito.isNull()); + eq(TbQueueCallback.EMPTY)); testBroadcastEntityStateChangeEventNever(createEntityId_NULL_UUID(new Tenant())); } else { Mockito.verify(tbClusterService, times(cntTime)).onTenantProfileChange(Mockito.argThat(matcherTenantProfile), diff --git a/application/src/test/java/org/thingsboard/server/edge/AlarmEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/AlarmEdgeTest.java index 24ed1a7801..3a88434103 100644 --- a/application/src/test/java/org/thingsboard/server/edge/AlarmEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/AlarmEdgeTest.java @@ -94,44 +94,26 @@ public class AlarmEdgeTest extends AbstractEdgeTest { Alarm savedAlarm = doPost("/api/alarm", alarm, Alarm.class); edgeImitator.ignoreType(AlarmCommentUpdateMsg.class); - // ack alarm + // ack alarm - send only by using push to edge node edgeImitator.expectMessageAmount(1); doPost("/api/alarm/" + savedAlarm.getUuidId() + "/ack"); - Assert.assertTrue(edgeImitator.waitForMessages()); - AbstractMessage latestMessage = edgeImitator.getLatestMessage(); - Assert.assertTrue(latestMessage instanceof AlarmUpdateMsg); - AlarmUpdateMsg alarmUpdateMsg = (AlarmUpdateMsg) latestMessage; - Assert.assertEquals(UpdateMsgType.ALARM_ACK_RPC_MESSAGE, alarmUpdateMsg.getMsgType()); - Alarm alarmMsg = JacksonUtil.fromString(alarmUpdateMsg.getEntity(), Alarm.class, true); - Assert.assertNotNull(alarmMsg); - Assert.assertEquals(savedAlarm.getType(), alarmMsg.getType()); - Assert.assertEquals(savedAlarm.getName(), alarmMsg.getName()); - Assert.assertEquals(AlarmStatus.ACTIVE_ACK, alarmMsg.getStatus()); + Assert.assertFalse(edgeImitator.waitForMessages(5)); - // clear alarm + // clear alarm - send only by using push to edge node edgeImitator.expectMessageAmount(1); doPost("/api/alarm/" + savedAlarm.getUuidId() + "/clear"); - Assert.assertTrue(edgeImitator.waitForMessages()); - latestMessage = edgeImitator.getLatestMessage(); - Assert.assertTrue(latestMessage instanceof AlarmUpdateMsg); - alarmUpdateMsg = (AlarmUpdateMsg) latestMessage; - Assert.assertEquals(UpdateMsgType.ALARM_CLEAR_RPC_MESSAGE, alarmUpdateMsg.getMsgType()); - alarmMsg = JacksonUtil.fromString(alarmUpdateMsg.getEntity(), Alarm.class, true); - Assert.assertNotNull(alarmMsg); - Assert.assertEquals(savedAlarm.getType(), alarmMsg.getType()); - Assert.assertEquals(savedAlarm.getName(), alarmMsg.getName()); - Assert.assertEquals(AlarmStatus.CLEARED_ACK, alarmMsg.getStatus()); + Assert.assertFalse(edgeImitator.waitForMessages(5)); // delete alarm edgeImitator.expectMessageAmount(1); doDelete("/api/alarm/" + savedAlarm.getUuidId()) .andExpect(status().isOk()); Assert.assertTrue(edgeImitator.waitForMessages()); - latestMessage = edgeImitator.getLatestMessage(); + AbstractMessage latestMessage = edgeImitator.getLatestMessage(); Assert.assertTrue(latestMessage instanceof AlarmUpdateMsg); - alarmUpdateMsg = (AlarmUpdateMsg) latestMessage; + AlarmUpdateMsg alarmUpdateMsg = (AlarmUpdateMsg) latestMessage; Assert.assertEquals(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE, alarmUpdateMsg.getMsgType()); - alarmMsg = JacksonUtil.fromString(alarmUpdateMsg.getEntity(), Alarm.class, true); + Alarm alarmMsg = JacksonUtil.fromString(alarmUpdateMsg.getEntity(), Alarm.class, true); Assert.assertNotNull(alarmMsg); Assert.assertEquals(savedAlarm.getType(), alarmMsg.getType()); Assert.assertEquals(savedAlarm.getName(), alarmMsg.getName()); @@ -252,4 +234,5 @@ public class AlarmEdgeTest extends AbstractEdgeTest { alarmComment.setCreatedTime(Uuids.unixTimestamp(uuid)); return alarmComment; } + } diff --git a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java index e34a61cfc3..4a303e5253 100644 --- a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java @@ -426,6 +426,9 @@ public class HashPartitionServiceTest { topicService); ReflectionTestUtils.setField(partitionService, "coreTopic", "tb.core"); ReflectionTestUtils.setField(partitionService, "corePartitions", 10); + ReflectionTestUtils.setField(partitionService, "cfEventTopic", "tb_cf_event"); + ReflectionTestUtils.setField(partitionService, "cfStateTopic", "tb_cf_state"); + ReflectionTestUtils.setField(partitionService, "cfPartitions", 10); ReflectionTestUtils.setField(partitionService, "vcTopic", "tb.vc"); ReflectionTestUtils.setField(partitionService, "vcPartitions", 10); ReflectionTestUtils.setField(partitionService, "hashFunctionName", hashFunctionName); diff --git a/application/src/test/java/org/thingsboard/server/service/apiusage/ApiUsageTest.java b/application/src/test/java/org/thingsboard/server/service/apiusage/ApiUsageTest.java new file mode 100644 index 0000000000..961185d3cd --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/apiusage/ApiUsageTest.java @@ -0,0 +1,144 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.TestPropertySource; +import org.thingsboard.server.common.data.ApiUsageStateValue; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.SaveDeviceWithCredentialsRequest; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.common.data.security.DeviceCredentialsType; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; +import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.controller.TbUrlConstants; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; + +import java.util.concurrent.TimeUnit; + +import static org.awaitility.Awaitility.await; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@DaoSqlTest +@TestPropertySource(properties = { + "usage.stats.report.enabled=true", + "usage.stats.report.interval=2", + "usage.stats.gauge_report_interval=1", +}) +public class ApiUsageTest extends AbstractControllerTest { + + private Tenant savedTenant; + private User tenantAdmin; + + private static final int MAX_DP_ENABLE_VALUE = 12; + private static final double WARN_THRESHOLD_VALUE = 0.5; + @Autowired + private ApiUsageStateService apiUsageStateService; + + @Before + public void beforeTest() throws Exception { + loginSysAdmin(); + + TenantProfile tenantProfile = createTenantProfile(); + TenantProfile savedTenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); + assertNotNull(savedTenantProfile); + + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + tenant.setTenantProfileId(savedTenantProfile.getId()); + savedTenant = saveTenant(tenant); + tenantId = savedTenant.getId(); + assertNotNull(savedTenant); + + tenantAdmin = new User(); + tenantAdmin.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin.setTenantId(savedTenant.getId()); + tenantAdmin.setEmail("tenant2@thingsboard.org"); + + tenantAdmin = createUserAndLogin(tenantAdmin, "testPassword1"); + } + + @Test + public void testTelemetryApiCall() throws Exception { + Device device = createDevice(); + assertNotNull(device); + String telemetryPayload = "{\"temperature\":25, \"humidity\":60}"; + String url = TbUrlConstants.TELEMETRY_URL_PREFIX + "/DEVICE/" + device.getId() + "/timeseries/ANY"; + + long VALUE_WARNING = (long) (MAX_DP_ENABLE_VALUE * WARN_THRESHOLD_VALUE) / 2; + + for (int i = 0; i < VALUE_WARNING; i++) { + doPostAsync(url, telemetryPayload, String.class, status().isOk()); + } + + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> assertEquals(ApiUsageStateValue.WARNING, apiUsageStateService.findTenantApiUsageState(tenantId).getDbStorageState())); + + long VALUE_DISABLE = (long) (MAX_DP_ENABLE_VALUE - (MAX_DP_ENABLE_VALUE * WARN_THRESHOLD_VALUE)) / 2; + + for (int i = 0; i < VALUE_DISABLE; i++) { + doPostAsync(url, telemetryPayload, String.class, status().isOk()); + } + + await().atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + assertEquals(ApiUsageStateValue.DISABLED, apiUsageStateService.findTenantApiUsageState(tenantId).getDbStorageState()); + }); + } + + private TenantProfile createTenantProfile() { + TenantProfile tenantProfile = new TenantProfile(); + tenantProfile.setName("Tenant Profile"); + tenantProfile.setDescription("Tenant Profile" + " Test"); + + TenantProfileData tenantProfileData = new TenantProfileData(); + DefaultTenantProfileConfiguration config = DefaultTenantProfileConfiguration.builder() + .maxDPStorageDays(MAX_DP_ENABLE_VALUE) + .warnThreshold(WARN_THRESHOLD_VALUE) + .build(); + + tenantProfileData.setConfiguration(config); + tenantProfile.setProfileData(tenantProfileData); + return tenantProfile; + } + + private Device createDevice() throws Exception { + String testToken = "TEST_TOKEN"; + + Device device = new Device(); + device.setName("My device"); + device.setType("default"); + device.setTenantId(tenantId); + + DeviceCredentials deviceCredentials = new DeviceCredentials(); + deviceCredentials.setCredentialsType(DeviceCredentialsType.ACCESS_TOKEN); + deviceCredentials.setCredentialsId(testToken); + + SaveDeviceWithCredentialsRequest saveRequest = new SaveDeviceWithCredentialsRequest(device, deviceCredentials); + + return readResponse(doPost("/api/device-with-credentials", saveRequest).andExpect(status().isOk()), Device.class); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateServiceTest.java b/application/src/test/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateServiceTest.java index 9701e57eb9..20f8aaf881 100644 --- a/application/src/test/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateServiceTest.java @@ -15,45 +15,114 @@ */ package org.thingsboard.server.service.apiusage; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.Spy; -import org.mockito.junit.jupiter.MockitoExtension; -import org.thingsboard.server.common.data.ApiUsageState; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.ApiUsageStateValue; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import java.util.UUID; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.mockito.Mockito.never; +import static org.junit.Assert.assertEquals; -@ExtendWith(MockitoExtension.class) -public class DefaultTbApiUsageStateServiceTest { +@DaoSqlTest +public class DefaultTbApiUsageStateServiceTest extends AbstractControllerTest { - @Mock - TenantApiUsageState tenantUsageStateMock; + @Autowired + DefaultTbApiUsageStateService service; - TenantId tenantId = TenantId.fromUUID(UUID.fromString("00797a3b-7aeb-4b5b-b57a-c2a810d0f112")); + @Autowired + private ApiUsageStateService apiUsageStateService; - @Spy - @InjectMocks - DefaultTbApiUsageStateService service; + private TenantId tenantId; + private Tenant savedTenant; + + private static final int MAX_ENABLE_VALUE = 5000; + private static final long VALUE_WARNING = 4500L; + private static final long VALUE_DISABLE = 5500L; + private static final double WARN_THRESHOLD_VALUE = 0.8; + + @Before + public void init() throws Exception { + loginSysAdmin(); - @BeforeEach - public void setUp() { + TenantProfile tenantProfile = createTenantProfile(); + TenantProfile savedTenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); + Assert.assertNotNull(savedTenantProfile); + + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + tenant.setTenantProfileId(savedTenantProfile.getId()); + savedTenant = saveTenant(tenant); + tenantId = savedTenant.getId(); + Assert.assertNotNull(savedTenant); } @Test - public void givenTenantIdFromEntityStatesMap_whenGetApiUsageState() { - service.myUsageStates.put(tenantId, tenantUsageStateMock); - ApiUsageState tenantUsageState = service.getApiUsageState(tenantId); - assertThat(tenantUsageState, is(tenantUsageStateMock.getApiUsageState())); - Mockito.verify(service, never()).getOrFetchState(tenantId, tenantId); + public void testProcess_transitionFromWarningToDisabled() { + TransportProtos.ToUsageStatsServiceMsg.Builder warningMsgBuilder = TransportProtos.ToUsageStatsServiceMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setCustomerIdMSB(0) + .setCustomerIdLSB(0) + .setServiceId("testService"); + + warningMsgBuilder.addValues(TransportProtos.UsageStatsKVProto.newBuilder() + .setKey(ApiUsageRecordKey.STORAGE_DP_COUNT.name()) + .setValue(VALUE_WARNING) + .build()); + + TransportProtos.ToUsageStatsServiceMsg warningStatsMsg = warningMsgBuilder.build(); + TbProtoQueueMsg warningMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), warningStatsMsg); + + service.process(warningMsg, TbCallback.EMPTY); + assertEquals(ApiUsageStateValue.WARNING, apiUsageStateService.findTenantApiUsageState(tenantId).getDbStorageState()); + + TransportProtos.ToUsageStatsServiceMsg.Builder disableMsgBuilder = TransportProtos.ToUsageStatsServiceMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setCustomerIdMSB(0) + .setCustomerIdLSB(0) + .setServiceId("testService"); + + disableMsgBuilder.addValues(TransportProtos.UsageStatsKVProto.newBuilder() + .setKey(ApiUsageRecordKey.STORAGE_DP_COUNT.name()) + .setValue(VALUE_DISABLE) + .build()); + + TransportProtos.ToUsageStatsServiceMsg disableStatsMsg = disableMsgBuilder.build(); + TbProtoQueueMsg disableMsg = new TbProtoQueueMsg<>(UUID.randomUUID(), disableStatsMsg); + + service.process(disableMsg, TbCallback.EMPTY); + assertEquals(ApiUsageStateValue.DISABLED, apiUsageStateService.findTenantApiUsageState(tenantId).getDbStorageState()); + } + + private TenantProfile createTenantProfile() { + TenantProfile tenantProfile = new TenantProfile(); + tenantProfile.setName("Tenant Profile"); + tenantProfile.setDescription("Tenant Profile" + " Test"); + + TenantProfileData tenantProfileData = new TenantProfileData(); + DefaultTenantProfileConfiguration config = DefaultTenantProfileConfiguration.builder() + .maxDPStorageDays(MAX_ENABLE_VALUE) + .warnThreshold(WARN_THRESHOLD_VALUE) + .build(); + + tenantProfileData.setConfiguration(config); + tenantProfile.setProfileData(tenantProfileData); + return tenantProfile; } } \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java new file mode 100644 index 0000000000..2cced15f59 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -0,0 +1,205 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.tbel.DefaultTbelInvokeService; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; +import org.thingsboard.server.service.cf.CalculatedFieldResult; + +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +@SpringBootTest(classes = DefaultTbelInvokeService.class) +public class ScriptCalculatedFieldStateTest { + + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("5b18e321-3327-4290-b996-d72a65e90382")); + private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); + private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); + + private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new DoubleDataEntry("assetHumidity", 43.0), 122L); + private final TsRollingArgumentEntry deviceTemperatureArgEntry = createRollingArgEntry(); + + private final long ts = System.currentTimeMillis(); + + private ScriptCalculatedFieldState state; + private CalculatedFieldCtx ctx; + + @Autowired + private TbelInvokeService tbelInvokeService; + + @MockBean + private ApiLimitService apiLimitService; + + @BeforeEach + void setUp() { + when(apiLimitService.getLimit(any(), any())).thenReturn(1000L); + ctx = new CalculatedFieldCtx(getCalculatedField(), tbelInvokeService, apiLimitService); + ctx.init(); + state = new ScriptCalculatedFieldState(ctx.getArgNames()); + } + + @Test + void testType() { + assertThat(state.getType()).isEqualTo(CalculatedFieldType.SCRIPT); + } + + @Test + void testUpdateState() { + state.arguments = new HashMap<>(Map.of("assetHumidity", assetHumidityArgEntry)); + + Map newArgs = Map.of("deviceTemperature", deviceTemperatureArgEntry); + boolean stateUpdated = state.updateState(newArgs); + + assertThat(stateUpdated).isTrue(); + assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( + Map.of( + "assetHumidity", assetHumidityArgEntry, + "deviceTemperature", deviceTemperatureArgEntry + ) + ); + } + + @Test + void testUpdateStateWhenUpdateExistingEntry() { + state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); + + SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(ts, new LongDataEntry("assetHumidity", 41L), 349L); + Map newArgs = Map.of("assetHumidity", newArgEntry); + boolean stateUpdated = state.updateState(newArgs); + + assertThat(stateUpdated).isTrue(); + assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( + Map.of( + "assetHumidity", newArgEntry, + "deviceTemperature", deviceTemperatureArgEntry + ) + ); + } + + @Test + void testPerformCalculation() throws ExecutionException, InterruptedException { + state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); + + CalculatedFieldResult result = state.performCalculation(ctx).get(); + + assertThat(result).isNotNull(); + Output output = getCalculatedFieldConfig().getOutput(); + assertThat(result.getType()).isEqualTo(output.getType()); + assertThat(result.getScope()).isEqualTo(output.getScope()); + assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("maxDeviceTemperature", 17.0, "assetHumidity", 43.0))); + } + + @Test + void testIsReadyWhenNotAllArgPresent() { + assertThat(state.isReady()).isFalse(); + } + + @Test + void testIsReadyWhenAllArgPresent() { + state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); + + assertThat(state.isReady()).isTrue(); + } + + @Test + void testIsReadyWhenEmptyEntryPresents() { + state.arguments = new HashMap<>(Map.of("deviceTemperature", new TsRollingArgumentEntry(5, 30000L), "assetHumidity", assetHumidityArgEntry)); + + assertThat(state.isReady()).isFalse(); + } + + private TsRollingArgumentEntry createRollingArgEntry() { + TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(5, 30000L); + long ts = System.currentTimeMillis(); + + TreeMap values = new TreeMap<>(); + values.put(ts - 40, 10.0); + values.put(ts - 30, 12.0); + values.put(ts - 20, 17.0); + + argumentEntry.setTsRecords(values); + return argumentEntry; + } + + private CalculatedField getCalculatedField() { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(TENANT_ID); + calculatedField.setEntityId(ASSET_ID); + calculatedField.setType(CalculatedFieldType.SCRIPT); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(getCalculatedFieldConfig()); + calculatedField.setVersion(1L); + return calculatedField; + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig() { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument1 = new Argument(); + argument1.setRefEntityId(DEVICE_ID); + ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("temperature", ArgumentType.TS_ROLLING, null); + argument1.setRefEntityKey(refEntityKey1); + argument1.setLimit(5); + argument1.setTimeWindow(30000L); + + Argument argument2 = new Argument(); + ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("humidity", ArgumentType.TS_LATEST, null); + argument1.setRefEntityKey(refEntityKey2); + + config.setArguments(Map.of("deviceTemperature", argument1, "assetHumidity", argument2)); + + config.setExpression("return {\"maxDeviceTemperature\": deviceTemperature.max(), \"assetHumidity\": assetHumidity.value}"); + + Output output = new Output(); + output.setType(OutputType.ATTRIBUTES); + output.setScope(AttributeScope.SERVER_SCOPE); + + config.setOutput(output); + + return config; + } + +} \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java new file mode 100644 index 0000000000..c06e835937 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -0,0 +1,228 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; +import org.thingsboard.server.service.cf.CalculatedFieldResult; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class SimpleCalculatedFieldStateTest { + + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("5b18e321-3327-4290-b996-d72a65e90382")); + private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); + private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); + + private final SingleValueArgumentEntry key1ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new LongDataEntry("key1", 11L), 145L); + private final SingleValueArgumentEntry key2ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 6, new LongDataEntry("key2", 15L), 165L); + private final SingleValueArgumentEntry key3ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 3, new LongDataEntry("key3", 23L), 184L); + + private SimpleCalculatedFieldState state; + private CalculatedFieldCtx ctx; + + @Mock + private ApiLimitService apiLimitService; + + @BeforeEach + void setUp() { + when(apiLimitService.getLimit(any(), any())).thenReturn(1000L); + ctx = new CalculatedFieldCtx(getCalculatedField(), null, apiLimitService); + ctx.init(); + state = new SimpleCalculatedFieldState(ctx.getArgNames()); + } + + @Test + void testType() { + assertThat(state.getType()).isEqualTo(CalculatedFieldType.SIMPLE); + } + + @Test + void testUpdateState() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry + )); + + Map newArgs = Map.of("key3", key3ArgEntry); + boolean stateUpdated = state.updateState(newArgs); + + assertThat(stateUpdated).isTrue(); + assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( + Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry, + "key3", key3ArgEntry + ) + ); + } + + @Test + void testUpdateStateWhenUpdateExistingEntry() { + state.arguments = new HashMap<>(Map.of("key1", key1ArgEntry)); + + SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis(), new LongDataEntry("key1", 18L), 190L); + Map newArgs = Map.of("key1", newArgEntry); + boolean stateUpdated = state.updateState(newArgs); + + assertThat(stateUpdated).isTrue(); + assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf(Map.of("key1", newArgEntry)); + } + + @Test + void testUpdateStateWhenRollingEntryPassed() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry + )); + + Map newArgs = Map.of("key3", new TsRollingArgumentEntry(10, 30000L)); + assertThatThrownBy(() -> state.updateState(newArgs)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Rolling argument entry is not supported for simple calculated fields."); + } + + @Test + void testPerformCalculation() throws ExecutionException, InterruptedException { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry, + "key3", key3ArgEntry + )); + + CalculatedFieldResult result = state.performCalculation(ctx).get(); + + assertThat(result).isNotNull(); + Output output = getCalculatedFieldConfig().getOutput(); + assertThat(result.getType()).isEqualTo(output.getType()); + assertThat(result.getScope()).isEqualTo(output.getScope()); + assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 49.0))); + } + + @Test + void testPerformCalculationWhenPassedNotNumber() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", new SingleValueArgumentEntry(System.currentTimeMillis() - 9, new StringDataEntry("key2", "string"), 124L), + "key3", key3ArgEntry + )); + + assertThatThrownBy(() -> state.performCalculation(ctx)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Argument 'key2' is not a number."); + } + + @Test + void testIsReadyWhenNotAllArgPresent() { + assertThat(state.isReady()).isFalse(); + } + + @Test + void testIsReadyWhenAllArgPresent() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry, + "key3", key3ArgEntry + )); + + assertThat(state.isReady()).isTrue(); + } + + @Test + void testIsReadyWhenEmptyEntryPresents() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry + )); + state.getArguments().put("key3", new SingleValueArgumentEntry()); + + assertThat(state.isReady()).isFalse(); + } + + private CalculatedField getCalculatedField() { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(TENANT_ID); + calculatedField.setEntityId(DEVICE_ID); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(getCalculatedFieldConfig()); + calculatedField.setVersion(1L); + return calculatedField; + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig() { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument1 = new Argument(); + argument1.setRefEntityId(ASSET_ID); + ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("temp1", ArgumentType.TS_LATEST, null); + argument1.setRefEntityKey(refEntityKey1); + + Argument argument2 = new Argument(); + argument2.setRefEntityId(ASSET_ID); + ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("temp2", ArgumentType.ATTRIBUTE, null); + argument2.setRefEntityKey(refEntityKey2); + + Argument argument3 = new Argument(); + argument3.setRefEntityId(ASSET_ID); + ReferencedEntityKey refEntityKey3 = new ReferencedEntityKey("temp3", ArgumentType.TS_LATEST, null); + argument3.setRefEntityKey(refEntityKey3); + + config.setArguments(Map.of("key1", argument1, "key2", argument2, "key3", argument3)); + + config.setExpression("key1 + key2 + key3"); + + Output output = new Output(); + output.setName("output"); + output.setType(OutputType.ATTRIBUTES); + output.setScope(AttributeScope.SERVER_SCOPE); + + config.setOutput(output); + + return config; + } + +} \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java new file mode 100644 index 0000000000..e83e30663d --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.kv.LongDataEntry; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class SingleValueArgumentEntryTest { + + private SingleValueArgumentEntry entry; + + private final long ts = System.currentTimeMillis(); + + @BeforeEach + void setUp() { + entry = new SingleValueArgumentEntry(ts, new LongDataEntry("key", 11L), 363L); + } + + @Test + void testArgumentEntryType() { + assertThat(entry.getType()).isEqualTo(ArgumentEntryType.SINGLE_VALUE); + } + + @Test + void testUpdateEntryWhenRollingEntryPassed() { + assertThatThrownBy(() -> entry.updateEntry(new TsRollingArgumentEntry(5, 30000L))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported argument entry type for single value argument entry: " + ArgumentEntryType.TS_ROLLING); + } + + @Test + void testUpdateEntryWithThaSameTs() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts, new LongDataEntry("key", 13L), 363L))).isFalse(); + } + + @Test + void testUpdateEntryWhenNewVersionIsNull() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 16, new LongDataEntry("key", 13L), null))).isTrue(); + assertThat(entry.getValue()).isEqualTo(13L); + assertThat(entry.getVersion()).isNull(); + } + + @Test + void testUpdateEntryWhenNewVersionIsGreaterThanCurrent() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 18L), 369L))).isTrue(); + assertThat(entry.getValue()).isEqualTo(18L); + assertThat(entry.getVersion()).isEqualTo(369L); + } + + @Test + void testUpdateEntryWhenNewVersionIsLessThanCurrent() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 18L), 234L))).isFalse(); + } + + @Test + void testUpdateEntryWhenValueWasNotChanged() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 11L), 237L))).isFalse(); + } +} \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java new file mode 100644 index 0000000000..b1f8063857 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java @@ -0,0 +1,123 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; + +import java.util.Map; +import java.util.TreeMap; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TsRollingArgumentEntryTest { + + private TsRollingArgumentEntry entry; + + private final long ts = System.currentTimeMillis(); + + @BeforeEach + void setUp() { + TreeMap values = new TreeMap<>(); + values.put(ts - 40, 10.0); + values.put(ts - 30, 12.0); + values.put(ts - 20, 17.0); + + entry = new TsRollingArgumentEntry(5, 30000L, values); + } + + @Test + void testArgumentEntryType() { + assertThat(entry.getType()).isEqualTo(ArgumentEntryType.TS_ROLLING); + } + + @Test + void testUpdateEntryWhenSingleValueEntryPassed() { + SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, new DoubleDataEntry("key", 23.0), 123L); + + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords()).hasSize(4); + assertThat(entry.getTsRecords().get(ts - 10)).isEqualTo(23.0); + } + + @Test + void testUpdateEntryWhenRollingEntryPassed() { + TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); + TreeMap values = new TreeMap<>(); + values.put(ts - 10, 7.0); + values.put(ts - 5, 1.0); + newEntry.setTsRecords(values); + + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords()).hasSize(5); + assertThat(entry.getTsRecords()).isEqualTo(Map.of( + ts - 40, 10.0, + ts - 30, 12.0, + ts - 20, 17.0, + ts - 10, 7.0, + ts - 5, 1.0 + )); + } + + @Test + void testUpdateEntryWhenValueIsNotNumber() { + SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, new StringDataEntry("key", "string"), 123L); + + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords().get(ts - 10)).isNaN(); + } + + @Test + void testUpdateEntryWhenOldTelemetry() { + TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); + TreeMap values = new TreeMap<>(); + values.put(ts - 40000, 4.0);// will not be used for calculation + values.put(ts - 45000, 2.0);// will not be used for calculation + values.put(ts - 5, 0.0); + newEntry.setTsRecords(values); + + entry = new TsRollingArgumentEntry(3, 30000L); + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords()).hasSize(1); + assertThat(entry.getTsRecords()).isEqualTo(Map.of( + ts - 5, 0.0 + )); + } + + @Test + void testPerformCalculationWhenArgumentsMoreThanLimit() { + TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); + TreeMap values = new TreeMap<>(); + values.put(ts - 20, 1000.0);// will not be used + values.put(ts - 18, 0.0); + values.put(ts - 16, 0.0); + values.put(ts - 14, 0.0); + newEntry.setTsRecords(values); + + entry = new TsRollingArgumentEntry(3, 30000L); + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords()).hasSize(3); + assertThat(entry.getTsRecords()).isEqualTo(Map.of( + ts - 18, 0.0, + ts - 16, 0.0, + ts - 14, 0.0 + )); + } + +} \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index 76d3ac1659..dc1f46eb3c 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -50,6 +50,7 @@ import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.objects.TelemetryEntityView; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.ApiUsageStateFilter; @@ -1767,7 +1768,7 @@ public class EntityServiceTest extends AbstractControllerTest { } } - List> timeseriesFutures = new ArrayList<>(); + List> timeseriesFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); timeseriesFutures.add(saveLongTimeseries(device.getId(), "temperature", temperatures.get(i))); @@ -2371,7 +2372,7 @@ public class EntityServiceTest extends AbstractControllerTest { return attributesService.save(tenantId, entityId, scope, Collections.singletonList(attr)); } - private ListenableFuture saveLongTimeseries(EntityId entityId, String key, Double value) { + private ListenableFuture saveLongTimeseries(EntityId entityId, String key, Double value) { TsKvEntity tsKv = new TsKvEntity(); tsKv.setStrKey(key); tsKv.setDoubleValue(value); diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java index b7f1f62582..ac85b48bc1 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java @@ -38,10 +38,17 @@ import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; +import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; +import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.entitiy.TbLogEntityActionService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; @@ -81,6 +88,20 @@ public class DefaultTbAlarmServiceTest { protected TbClusterService tbClusterService; @MockBean private EntitiesVersionControlService vcService; + @MockBean + private AccessControlService accessControlService; + @MockBean + private TenantService tenantService; + @MockBean + private AssetService assetService; + @MockBean + private DeviceService deviceService; + @MockBean + private AssetProfileService assetProfileService; + @MockBean + private DeviceProfileService deviceProfileService; + @MockBean + private EntityService entityService; @SpyBean DefaultTbAlarmService service; diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java index 0196857b1e..3c00c2957a 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java @@ -35,10 +35,17 @@ import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.dao.alarm.AlarmCommentService; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; +import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.entitiy.TbLogEntityActionService; import org.thingsboard.server.service.entitiy.alarm.DefaultTbAlarmCommentService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; import java.util.UUID; @@ -72,6 +79,20 @@ public class DefaultTbAlarmCommentServiceTest { protected CustomerService customerService; @MockBean protected TbClusterService tbClusterService; + @MockBean + private AccessControlService accessControlService; + @MockBean + private TenantService tenantService; + @MockBean + private AssetService assetService; + @MockBean + private DeviceService deviceService; + @MockBean + private AssetProfileService assetProfileService; + @MockBean + private DeviceProfileService deviceProfileService; + @MockBean + private EntityService entityService; @SpyBean DefaultTbAlarmCommentService service; diff --git a/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java b/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java index 9b6c5e6779..40369e231a 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java @@ -44,6 +44,7 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueCallback; @@ -102,6 +103,8 @@ public class DefaultTbClusterServiceTest { protected TbRuleEngineProducerService ruleEngineProducerService; @MockBean protected TbTransactionalCache edgeCache; + @MockBean + protected CalculatedFieldService calculatedFieldService; @SpyBean protected TopicService topicService; diff --git a/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java b/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java index 2ec917c81a..fe416eacd5 100644 --- a/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java @@ -16,15 +16,22 @@ package org.thingsboard.server.service.resource.sql; import com.datastax.oss.driver.api.core.uuid.Uuids; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.jupiter.api.Assertions; import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.DashboardInfo; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityInfo; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TbResourceDeleteResult; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.TbResourceInfoFilter; import org.thingsboard.server.common.data.Tenant; @@ -36,10 +43,14 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.widget.WidgetTypeDetails; +import org.thingsboard.server.common.data.widget.WidgetTypeInfo; import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.service.resource.TbResourceService; import java.util.ArrayList; @@ -108,6 +119,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { ""; private static final String DEFAULT_FILE_NAME = "test.jks"; + private static final String JS_FILE_NAME = "test.js"; private static final String TEST_BASE64_DATA = "77u/PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPCEtLQpGSUxFIElORk9STUFUSU9OCgpPTUEgUGVybWFuZW50IERvY3VtZW50CiAgIEZpbGU6IE9NQS1TVVAtTHdNMk1fQmluYXJ5QXBwRGF0YUNvbnRhaW5lci1WMV8wXzEtMjAxOTAyMjEtQQogICBUeXBlOiB4bWwKClB1YmxpYyBSZWFjaGFibGUgSW5mb3JtYXRpb24KICAgUGF0aDogaHR0cDovL3d3dy5vcGVubW9iaWxlYWxsaWFuY2Uub3JnL3RlY2gvcHJvZmlsZXMKICAgTmFtZTogTHdNMk1fQmluYXJ5QXBwRGF0YUNvbnRhaW5lci12MV8wXzEueG1sCgpOT1JNQVRJVkUgSU5GT1JNQVRJT04KCiAgSW5mb3JtYXRpb24gYWJvdXQgdGhpcyBmaWxlIGNhbiBiZSBmb3VuZCBpbiB0aGUgbGF0ZXN0IHJldmlzaW9uIG9mCgogIE9NQS1UUy1MV00yTV9CaW5hcnlBcHBEYXRhQ29udGFpbmVyLVYxXzBfMQoKICBUaGlzIGlzIGF2YWlsYWJsZSBhdCBodHRwOi8vd3d3Lm9wZW5tb2JpbGVhbGxpYW5jZS5vcmcvCgogIFNlbmQgY29tbWVudHMgdG8gaHR0cHM6Ly9naXRodWIuY29tL09wZW5Nb2JpbGVBbGxpYW5jZS9PTUFfTHdNMk1fZm9yX0RldmVsb3BlcnMvaXNzdWVzCgpDSEFOR0UgSElTVE9SWQoKMTUwNjIwMTggU3RhdHVzIGNoYW5nZWQgdG8gQXBwcm92ZWQgYnkgRE0sIERvYyBSZWYgIyBPTUEtRE0mU0UtMjAxOC0wMDYxLUlOUF9MV00yTV9BUFBEQVRBX1YxXzBfRVJQX2Zvcl9maW5hbF9BcHByb3ZhbAoyMTAyMjAxOSBTdGF0dXMgY2hhbmdlZCB0byBBcHByb3ZlZCBieSBJUFNPLCBEb2MgUmVmICMgT01BLUlQU08tMjAxOS0wMDI1LUlOUF9Md00yTV9PYmplY3RfQXBwX0RhdGFfQ29udGFpbmVyXzFfMF8xX2Zvcl9GaW5hbF9BcHByb3ZhbAoKTEVHQUwgRElTQ0xBSU1FUgoKQ29weXJpZ2h0IDIwMTkgT3BlbiBNb2JpbGUgQWxsaWFuY2UuCgpSZWRpc3RyaWJ1dGlvbiBhbmQgdXNlIGluIHNvdXJjZSBhbmQgYmluYXJ5IGZvcm1zLCB3aXRoIG9yIHdpdGhvdXQKbW9kaWZpY2F0aW9uLCBhcmUgcGVybWl0dGVkIHByb3ZpZGVkIHRoYXQgdGhlIGZvbGxvd2luZyBjb25kaXRpb25zCmFyZSBtZXQ6CgoxLiBSZWRpc3RyaWJ1dGlvbnMgb2Ygc291cmNlIGNvZGUgbXVzdCByZXRhaW4gdGhlIGFib3ZlIGNvcHlyaWdodApub3RpY2UsIHRoaXMgbGlzdCBvZiBjb25kaXRpb25zIGFuZCB0aGUgZm9sbG93aW5nIGRpc2NsYWltZXIuCjIuIFJlZGlzdHJpYnV0aW9ucyBpbiBiaW5hcnkgZm9ybSBtdXN0IHJlcHJvZHVjZSB0aGUgYWJvdmUgY29weXJpZ2h0Cm5vdGljZSwgdGhpcyBsaXN0IG9mIGNvbmRpdGlvbnMgYW5kIHRoZSBmb2xsb3dpbmcgZGlzY2xhaW1lciBpbiB0aGUKZG9jdW1lbnRhdGlvbiBhbmQvb3Igb3RoZXIgbWF0ZXJpYWxzIHByb3ZpZGVkIHdpdGggdGhlIGRpc3RyaWJ1dGlvbi4KMy4gTmVpdGhlciB0aGUgbmFtZSBvZiB0aGUgY29weXJpZ2h0IGhvbGRlciBub3IgdGhlIG5hbWVzIG9mIGl0cwpjb250cmlidXRvcnMgbWF5IGJlIHVzZWQgdG8gZW5kb3JzZSBvciBwcm9tb3RlIHByb2R1Y3RzIGRlcml2ZWQKZnJvbSB0aGlzIHNvZnR3YXJlIHdpdGhvdXQgc3BlY2lmaWMgcHJpb3Igd3JpdHRlbiBwZXJtaXNzaW9uLgoKVEhJUyBTT0ZUV0FSRSBJUyBQUk9WSURFRCBCWSBUSEUgQ09QWVJJR0hUIEhPTERFUlMgQU5EIENPTlRSSUJVVE9SUwoiQVMgSVMiIEFORCBBTlkgRVhQUkVTUyBPUiBJTVBMSUVEIFdBUlJBTlRJRVMsIElOQ0xVRElORywgQlVUIE5PVApMSU1JVEVEIFRPLCBUSEUgSU1QTElFRCBXQVJSQU5USUVTIE9GIE1FUkNIQU5UQUJJTElUWSBBTkQgRklUTkVTUwpGT1IgQSBQQVJUSUNVTEFSIFBVUlBPU0UgQVJFIERJU0NMQUlNRUQuIElOIE5PIEVWRU5UIFNIQUxMIFRIRQpDT1BZUklHSFQgSE9MREVSIE9SIENPTlRSSUJVVE9SUyBCRSBMSUFCTEUgRk9SIEFOWSBESVJFQ1QsIElORElSRUNULApJTkNJREVOVEFMLCBTUEVDSUFMLCBFWEVNUExBUlksIE9SIENPTlNFUVVFTlRJQUwgREFNQUdFUyAoSU5DTFVESU5HLApCVVQgTk9UIExJTUlURUQgVE8sIFBST0NVUkVNRU5UIE9GIFNVQlNUSVRVVEUgR09PRFMgT1IgU0VSVklDRVM7CkxPU1MgT0YgVVNFLCBEQVRBLCBPUiBQUk9GSVRTOyBPUiBCVVNJTkVTUyBJTlRFUlJVUFRJT04pIEhPV0VWRVIKQ0FVU0VEIEFORCBPTiBBTlkgVEhFT1JZIE9GIExJQUJJTElUWSwgV0hFVEhFUiBJTiBDT05UUkFDVCwgU1RSSUNUCkxJQUJJTElUWSwgT1IgVE9SVCAoSU5DTFVESU5HIE5FR0xJR0VOQ0UgT1IgT1RIRVJXSVNFKSBBUklTSU5HIElOCkFOWSBXQVkgT1VUIE9GIFRIRSBVU0UgT0YgVEhJUyBTT0ZUV0FSRSwgRVZFTiBJRiBBRFZJU0VEIE9GIFRIRQpQT1NTSUJJTElUWSBPRiBTVUNIIERBTUFHRS4KClRoZSBhYm92ZSBsaWNlbnNlIGlzIHVzZWQgYXMgYSBsaWNlbnNlIHVuZGVyIGNvcHlyaWdodCBvbmx5LiBQbGVhc2UKcmVmZXJlbmNlIHRoZSBPTUEgSVBSIFBvbGljeSBmb3IgcGF0ZW50IGxpY2Vuc2luZyB0ZXJtczoKaHR0cHM6Ly93d3cub21hc3BlY3dvcmtzLm9yZy9hYm91dC9pbnRlbGxlY3R1YWwtcHJvcGVydHktcmlnaHRzLwoKLS0+CjxMV00yTSB4bWxuczp4c2k9Imh0dHA6Ly93d3cudzMub3JnLzIwMDEvWE1MU2NoZW1hLWluc3RhbmNlIiB4c2k6bm9OYW1lc3BhY2VTY2hlbWFMb2NhdGlvbj0iaHR0cDovL29wZW5tb2JpbGVhbGxpYW5jZS5vcmcvdGVjaC9wcm9maWxlcy9MV00yTS54c2QiPgoJPE9iamVjdCBPYmplY3RUeXBlPSJNT0RlZmluaXRpb24iPgoJCTxOYW1lPkJpbmFyeUFwcERhdGFDb250YWluZXI8L05hbWU+CgkJPERlc2NyaXB0aW9uMT48IVtDREFUQVtUaGlzIEx3TTJNIE9iamVjdHMgcHJvdmlkZXMgdGhlIGFwcGxpY2F0aW9uIHNlcnZpY2UgZGF0YSByZWxhdGVkIHRvIGEgTHdNMk0gU2VydmVyLCBlZy4gV2F0ZXIgbWV0ZXIgZGF0YS4gClRoZXJlIGFyZSBzZXZlcmFsIG1ldGhvZHMgdG8gY3JlYXRlIGluc3RhbmNlIHRvIGluZGljYXRlIHRoZSBtZXNzYWdlIGRpcmVjdGlvbiBiYXNlZCBvbiB0aGUgbmVnb3RpYXRpb24gYmV0d2VlbiBBcHBsaWNhdGlvbiBhbmQgTHdNMk0uIFRoZSBDbGllbnQgYW5kIFNlcnZlciBzaG91bGQgbmVnb3RpYXRlIHRoZSBpbnN0YW5jZShzKSB1c2VkIHRvIGV4Y2hhbmdlIHRoZSBkYXRhLiBGb3IgZXhhbXBsZToKIC0gVXNpbmcgYSBzaW5nbGUgaW5zdGFuY2UgZm9yIGJvdGggZGlyZWN0aW9ucyBjb21tdW5pY2F0aW9uLCBmcm9tIENsaWVudCB0byBTZXJ2ZXIgYW5kIGZyb20gU2VydmVyIHRvIENsaWVudC4KIC0gVXNpbmcgYW4gaW5zdGFuY2UgZm9yIGNvbW11bmljYXRpb24gZnJvbSBDbGllbnQgdG8gU2VydmVyIGFuZCBhbm90aGVyIG9uZSBmb3IgY29tbXVuaWNhdGlvbiBmcm9tIFNlcnZlciB0byBDbGllbnQKIC0gVXNpbmcgc2V2ZXJhbCBpbnN0YW5jZXMKXV0+PC9EZXNjcmlwdGlvbjE+CgkJPE9iamVjdElEPjE5PC9PYmplY3RJRD4KCQk8T2JqZWN0VVJOPnVybjpvbWE6bHdtMm06b21hOjE5PC9PYmplY3RVUk4+CgkJPExXTTJNVmVyc2lvbj4xLjA8L0xXTTJNVmVyc2lvbj4KCQk8T2JqZWN0VmVyc2lvbj4xLjA8L09iamVjdFZlcnNpb24+CgkJPE11bHRpcGxlSW5zdGFuY2VzPk11bHRpcGxlPC9NdWx0aXBsZUluc3RhbmNlcz4KCQk8TWFuZGF0b3J5Pk9wdGlvbmFsPC9NYW5kYXRvcnk+CgkJPFJlc291cmNlcz4KCQkJPEl0ZW0gSUQ9IjAiPjxOYW1lPkRhdGE8L05hbWU+CgkJCQk8T3BlcmF0aW9ucz5SVzwvT3BlcmF0aW9ucz4KCQkJCTxNdWx0aXBsZUluc3RhbmNlcz5NdWx0aXBsZTwvTXVsdGlwbGVJbnN0YW5jZXM+CgkJCQk8TWFuZGF0b3J5Pk1hbmRhdG9yeTwvTWFuZGF0b3J5PgoJCQkJPFR5cGU+T3BhcXVlPC9UeXBlPgoJCQkJPFJhbmdlRW51bWVyYXRpb24gLz4KCQkJCTxVbml0cyAvPgoJCQkJPERlc2NyaXB0aW9uPjwhW0NEQVRBW0luZGljYXRlcyB0aGUgYXBwbGljYXRpb24gZGF0YSBjb250ZW50Ll1dPjwvRGVzY3JpcHRpb24+CgkJCTwvSXRlbT4KCQkJPEl0ZW0gSUQ9IjEiPjxOYW1lPkRhdGEgUHJpb3JpdHk8L05hbWU+CgkJCQk8T3BlcmF0aW9ucz5SVzwvT3BlcmF0aW9ucz4KCQkJCTxNdWx0aXBsZUluc3RhbmNlcz5TaW5nbGU8L011bHRpcGxlSW5zdGFuY2VzPgoJCQkJPE1hbmRhdG9yeT5PcHRpb25hbDwvTWFuZGF0b3J5PgoJCQkJPFR5cGU+SW50ZWdlcjwvVHlwZT4KCQkJCTxSYW5nZUVudW1lcmF0aW9uPjEgYnl0ZXM8L1JhbmdlRW51bWVyYXRpb24+CgkJCQk8VW5pdHMgLz4KCQkJCTxEZXNjcmlwdGlvbj48IVtDREFUQVtJbmRpY2F0ZXMgdGhlIEFwcGxpY2F0aW9uIGRhdGEgcHJpb3JpdHk6CjA6SW1tZWRpYXRlCjE6QmVzdEVmZm9ydAoyOkxhdGVzdAozLTEwMDogUmVzZXJ2ZWQgZm9yIGZ1dHVyZSB1c2UuCjEwMS0yNTQ6IFByb3ByaWV0YXJ5IG1vZGUuXV0+PC9EZXNjcmlwdGlvbj4KCQkJPC9JdGVtPgoJCQk8SXRlbSBJRD0iMiI+PE5hbWU+RGF0YSBDcmVhdGlvbiBUaW1lPC9OYW1lPgoJCQkJPE9wZXJhdGlvbnM+Ulc8L09wZXJhdGlvbnM+CgkJCQk8TXVsdGlwbGVJbnN0YW5jZXM+U2luZ2xlPC9NdWx0aXBsZUluc3RhbmNlcz4KCQkJCTxNYW5kYXRvcnk+T3B0aW9uYWw8L01hbmRhdG9yeT4KCQkJCTxUeXBlPlRpbWU8L1R5cGU+CgkJCQk8UmFuZ2VFbnVtZXJhdGlvbiAvPgoJCQkJPFVuaXRzIC8+CgkJCQk8RGVzY3JpcHRpb24+PCFbQ0RBVEFbSW5kaWNhdGVzIHRoZSBEYXRhIGluc3RhbmNlIGNyZWF0aW9uIHRpbWVzdGFtcC5dXT48L0Rlc2NyaXB0aW9uPgoJCQk8L0l0ZW0+CgkJCTxJdGVtIElEPSIzIj48TmFtZT5EYXRhIERlc2NyaXB0aW9uPC9OYW1lPgoJCQkJPE9wZXJhdGlvbnM+Ulc8L09wZXJhdGlvbnM+CgkJCQk8TXVsdGlwbGVJbnN0YW5jZXM+U2luZ2xlPC9NdWx0aXBsZUluc3RhbmNlcz4KCQkJCTxNYW5kYXRvcnk+T3B0aW9uYWw8L01hbmRhdG9yeT4KCQkJCTxUeXBlPlN0cmluZzwvVHlwZT4KCQkJCTxSYW5nZUVudW1lcmF0aW9uPjMyIGJ5dGVzPC9SYW5nZUVudW1lcmF0aW9uPgoJCQkJPFVuaXRzIC8+CgkJCQk8RGVzY3JpcHRpb24+PCFbQ0RBVEFbSW5kaWNhdGVzIHRoZSBkYXRhIGRlc2NyaXB0aW9uLgplLmcuICJtZXRlciByZWFkaW5nIi5dXT48L0Rlc2NyaXB0aW9uPgoJCQk8L0l0ZW0+CgkJCTxJdGVtIElEPSI0Ij48TmFtZT5EYXRhIEZvcm1hdDwvTmFtZT4KCQkJCTxPcGVyYXRpb25zPlJXPC9PcGVyYXRpb25zPgoJCQkJPE11bHRpcGxlSW5zdGFuY2VzPlNpbmdsZTwvTXVsdGlwbGVJbnN0YW5jZXM+CgkJCQk8TWFuZGF0b3J5Pk9wdGlvbmFsPC9NYW5kYXRvcnk+CgkJCQk8VHlwZT5TdHJpbmc8L1R5cGU+CgkJCQk8UmFuZ2VFbnVtZXJhdGlvbj4zMiBieXRlczwvUmFuZ2VFbnVtZXJhdGlvbj4KCQkJCTxVbml0cyAvPgoJCQkJPERlc2NyaXB0aW9uPjwhW0NEQVRBW0luZGljYXRlcyB0aGUgZm9ybWF0IG9mIHRoZSBBcHBsaWNhdGlvbiBEYXRhLgplLmcuIFlHLU1ldGVyLVdhdGVyLVJlYWRpbmcKVVRGOC1zdHJpbmcKXV0+PC9EZXNjcmlwdGlvbj4KCQkJPC9JdGVtPgoJCQk8SXRlbSBJRD0iNSI+PE5hbWU+QXBwIElEPC9OYW1lPgoJCQkJPE9wZXJhdGlvbnM+Ulc8L09wZXJhdGlvbnM+CgkJCQk8TXVsdGlwbGVJbnN0YW5jZXM+U2luZ2xlPC9NdWx0aXBsZUluc3RhbmNlcz4KCQkJCTxNYW5kYXRvcnk+T3B0aW9uYWw8L01hbmRhdG9yeT4KCQkJCTxUeXBlPkludGVnZXI8L1R5cGU+CgkJCQk8UmFuZ2VFbnVtZXJhdGlvbj4yIGJ5dGVzPC9SYW5nZUVudW1lcmF0aW9uPgoJCQkJPFVuaXRzIC8+CgkJCQk8RGVzY3JpcHRpb24+PCFbQ0RBVEFbSW5kaWNhdGVzIHRoZSBkZXN0aW5hdGlvbiBBcHBsaWNhdGlvbiBJRC5dXT48L0Rlc2NyaXB0aW9uPgoJCQk8L0l0ZW0+PC9SZXNvdXJjZXM+CgkJPERlc2NyaXB0aW9uMj48IVtDREFUQVtdXT48L0Rlc2NyaXB0aW9uMj4KCTwvT2JqZWN0Pgo8L0xXTTJNPgo="; private static final byte[] TEST_DATA = Base64.getDecoder().decode(TEST_BASE64_DATA); @@ -119,6 +131,10 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { private ResourceService resourceService; @Autowired private TbResourceService tbResourceService; + @Autowired + private WidgetTypeService widgetTypeService; + @Autowired + private DashboardService dashboardService; private Tenant savedTenant; private User tenantAdmin; @@ -141,6 +157,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { tenantAdmin.setLastName("Downs"); tenantAdmin = createUserAndLogin(tenantAdmin, "testPassword1"); + } @After @@ -244,7 +261,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { assertEquals(title, foundResource.getTitle()); assertArrayEquals(foundResource.getData(), TEST_DATA); - tbResourceService.delete(foundResource, null); + tbResourceService.delete(foundResource, true, null); } @Test @@ -267,7 +284,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { assertEquals("0_1.0", foundResource.getResourceKey()); assertArrayEquals(foundResource.getData(), LWM2M_TEST_MODEL.getBytes()); - tbResourceService.delete(foundResource, null); + tbResourceService.delete(savedResource, true, null); } @Test @@ -281,8 +298,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { assertEquals(TenantId.SYS_TENANT_ID, savedResource.getTenantId()); - TbResource foundResource = resourceService.findResourceById(tenantId, savedResource.getId()); - tbResourceService.delete(foundResource, null); + tbResourceService.delete(savedResource, true, null); } @Test @@ -361,7 +377,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { Assert.assertNotNull(foundResource); assertEquals(savedResource, new TbResourceInfo(foundResource)); assertArrayEquals(TEST_DATA, foundResource.getData()); - tbResourceService.delete(foundResource, null); + tbResourceService.delete(foundResource, true, null); } @Test @@ -378,22 +394,207 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { Assert.assertNotNull(foundResource); assertEquals(savedResource, new TbResourceInfo(foundResource)); assertArrayEquals(TEST_DATA, foundResource.getData()); - tbResourceService.delete(foundResource, null); + tbResourceService.delete(foundResource, true, null); } @Test public void testDeleteResource() throws Exception { TbResource resource = new TbResource(); - resource.setResourceType(ResourceType.JKS); + resource.setResourceType(ResourceType.JS_MODULE); resource.setTitle("My resource"); resource.setFileName(DEFAULT_FILE_NAME); resource.setData(TEST_DATA); TbResourceInfo savedResource = tbResourceService.save(resource); - TbResource foundResource = resourceService.findResourceById(tenantId, savedResource.getId()); + TbResource foundResource = resourceService.findResourceById(savedTenant.getId(), savedResource.getId()); + Assert.assertNotNull(foundResource); + tbResourceService.delete(savedResource, true, null); + foundResource = resourceService.findResourceById(savedTenant.getId(), savedResource.getId()); + Assert.assertNull(foundResource); + } + + @Test + public void testUnForceDeleteResourceAssignWithWidget() throws Exception { + TbResource resource = new TbResource(); + resource.setResourceType(ResourceType.JS_MODULE); + resource.setTitle("My resource"); + resource.setFileName(JS_FILE_NAME); + resource.setTenantId(savedTenant.getId()); + resource.setData(TEST_DATA); + TbResourceInfo savedResource = tbResourceService.save(resource); + TbResource foundResource = resourceService.findResourceById(savedTenant.getId(), savedResource.getId()); + Assert.assertNotNull(foundResource); + String link = DataConstants.TB_RESOURCE_PREFIX + resource.getLink(); + + WidgetTypeDetails widgetTypeDetails = new WidgetTypeDetails(); + widgetTypeDetails.setTenantId(savedTenant.getId()); + widgetTypeDetails.setDescriptor(JacksonUtil.newObjectNode() + .put("sizeX", 3) + .put("sizeY", 3) + .put("resource", link) + .put("templateCss", "") + .put("controllerScript", "self.onInit = function() {\n self.ctx.$scope.actionWidget.onInit();\n}\n\nself.typeParameters = function() {\n return {\n previewWidth: '300px',\n previewHeight: '320px',\n embedTitlePanel: true,\n targetDeviceOptional: true,\n displayRpcMessageToast: false\n };\n};\n\nself.onDestroy = function() {\n}") + .put("settingsSchema", "") + .put("dataKeySettingsSchema", "{}\n") + .put("settingsDirective", "tb-scada-symbol-widget-settings") + .put("hasBasicMode", true) + .put("basicModeDirective", "tb-scada-symbol-basic-config")); + widgetTypeDetails.setName("Widget Type"); + + WidgetTypeDetails savedWidgetType = widgetTypeService.saveWidgetType(widgetTypeDetails); + WidgetTypeDetails foundWidgetType = widgetTypeService.findWidgetTypeDetailsById(savedTenant.getId(), savedWidgetType.getId()); + String resourceLink = foundWidgetType.getDescriptor().get("resource").asText(); + Assertions.assertNotNull(resourceLink); + Assert.assertEquals(resourceLink, link); + + TbResourceDeleteResult result = tbResourceService.delete(savedResource, false, null); + Assert.assertNotNull(result); + Assert.assertFalse(result.isSuccess()); + Assert.assertFalse(result.getReferences().isEmpty()); + Assert.assertEquals(1, result.getReferences().size()); + + WidgetTypeInfo widgetTypeInfo = (WidgetTypeInfo) result.getReferences().get(EntityType.WIDGET_TYPE.name()).get(0); + WidgetTypeInfo foundWidgetTypeInfo = new WidgetTypeInfo(foundWidgetType); + Assert.assertNotNull(widgetTypeInfo); + Assert.assertNotNull(foundWidgetTypeInfo); + Assert.assertEquals(widgetTypeInfo, foundWidgetTypeInfo); + + TbResourceInfo foundResourceInfo = resourceService.findResourceInfoById(savedTenant.getId(), savedResource.getId()); + Assert.assertNotNull(foundResource); + Assert.assertEquals(savedResource, foundResourceInfo); + } + + @Test + public void testForceDeleteResourceAssignWithWidget() throws Exception { + TbResource resource = new TbResource(); + resource.setResourceType(ResourceType.JS_MODULE); + resource.setTitle("My resource"); + resource.setFileName(JS_FILE_NAME); + resource.setTenantId(savedTenant.getId()); + resource.setData(TEST_DATA); + TbResourceInfo savedResource = tbResourceService.save(resource); + TbResource foundResource = resourceService.findResourceById(savedTenant.getId(), savedResource.getId()); + Assert.assertNotNull(foundResource); + String link = DataConstants.TB_RESOURCE_PREFIX + resource.getLink(); + + WidgetTypeDetails widgetTypeDetails = new WidgetTypeDetails(); + widgetTypeDetails.setTenantId(savedTenant.getId()); + widgetTypeDetails.setDescriptor(JacksonUtil.newObjectNode() + .put("type", "rpc") + .put("sizeX", 3) + .put("sizeY", 3) + .put("resource", link) + .put("templateCss", "") + .put("controllerScript", "self.onInit = function() {\n self.ctx.$scope.actionWidget.onInit();\n}\n\nself.typeParameters = function() {\n return {\n previewWidth: '300px',\n previewHeight: '320px',\n embedTitlePanel: true,\n targetDeviceOptional: true,\n displayRpcMessageToast: false\n };\n};\n\nself.onDestroy = function() {\n}") + .put("settingsSchema", "") + .put("dataKeySettingsSchema", "{}\n") + .put("settingsDirective", "tb-scada-symbol-widget-settings") + .put("hasBasicMode", true) + .put("basicModeDirective", "tb-scada-symbol-basic-config")); + widgetTypeDetails.setName("Widget Type"); + + WidgetTypeDetails savedWidgetType = widgetTypeService.saveWidgetType(widgetTypeDetails); + WidgetTypeDetails foundWidgetType = widgetTypeService.findWidgetTypeDetailsById(savedTenant.getId(), savedWidgetType.getId()); + String resourceLink = foundWidgetType.getDescriptor().get("resource").asText(); + Assertions.assertNotNull(resourceLink); + Assert.assertEquals(resourceLink, link); + + TbResourceDeleteResult result = tbResourceService.delete(savedResource, true, null); + Assert.assertNotNull(result); + Assert.assertTrue(result.isSuccess()); + Assert.assertNull(result.getReferences()); + + foundResource = resourceService.findResourceById(savedTenant.getId(), savedResource.getId()); + Assert.assertNull(foundResource); + } + + @Test + public void testUnForceDeleteResourceAssignWithDashboard() throws Exception { + TbResource resource = new TbResource(); + resource.setResourceType(ResourceType.JS_MODULE); + resource.setTitle("My resource"); + resource.setFileName(JS_FILE_NAME); + resource.setTenantId(savedTenant.getId()); + resource.setData(TEST_DATA); + TbResourceInfo savedResource = tbResourceService.save(resource); + TbResource foundResource = resourceService.findResourceById(savedTenant.getId(), savedResource.getId()); + Assert.assertNotNull(foundResource); + String link = DataConstants.TB_RESOURCE_PREFIX + resource.getLink(); + + Dashboard dashboard = new Dashboard(); + dashboard.setTitle("My dashboard"); + dashboard.setTenantId(savedTenant.getId()); + dashboard.setConfiguration(JacksonUtil.newObjectNode() + .put("widgets", """ + {"xxx": + {"config":{"actions":{"elementClick":[ + {"customResources":[{"url":{"entityType":"TB_RESOURCE","id": + "tb-resource;/api/resource/js_module/tenant/gateway-management-extension.js"},"isModule":true}, + {"url":"tb-resource;/api/resource/js_module/tenant/gateway-management-extension.js","isModule":true}]}]}}}} + """) + .put("someResource", link)); + + Dashboard savedDashboard = dashboardService.saveDashboard(dashboard); + Dashboard foundDashboard = dashboardService.findDashboardById(savedTenant.getId(), savedDashboard.getId()); + String resourceLink = foundDashboard.getConfiguration().get("someResource").asText(); + Assertions.assertNotNull(resourceLink); + Assert.assertEquals(resourceLink, link); + + TbResourceDeleteResult result = tbResourceService.delete(savedResource, false, null); + Assert.assertNotNull(result); + Assert.assertFalse(result.isSuccess()); + Assert.assertNotNull(result.getReferences()); + Assert.assertEquals(1, result.getReferences().size()); + + DashboardInfo dashboardInfo = (DashboardInfo) result.getReferences().get(EntityType.DASHBOARD.name()).get(0); + DashboardInfo foundDashboardInfo = dashboardService.findDashboardInfoById(savedTenant.getId(), savedDashboard.getId()); + Assert.assertNotNull(dashboardInfo); + Assert.assertNotNull(foundDashboardInfo); + Assert.assertEquals(foundDashboardInfo, dashboardInfo); + + foundResource = resourceService.findResourceById(savedTenant.getId(), savedResource.getId()); + Assert.assertNotNull(foundResource); + Assert.assertEquals(savedDashboard, foundDashboard); + } + + @Test + public void testForceDeleteResourceAssignWithDashboard() throws Exception { + TbResource resource = new TbResource(); + resource.setResourceType(ResourceType.JS_MODULE); + resource.setTitle("My resource"); + resource.setFileName(JS_FILE_NAME); + resource.setTenantId(savedTenant.getId()); + resource.setData(TEST_DATA); + TbResourceInfo savedResource = tbResourceService.save(resource); + TbResource foundResource = resourceService.findResourceById(savedTenant.getId(), savedResource.getId()); Assert.assertNotNull(foundResource); - tbResourceService.delete(foundResource, null); - foundResource = resourceService.findResourceById(tenantId, savedResource.getId()); + String link = DataConstants.TB_RESOURCE_PREFIX + resource.getLink(); + + Dashboard dashboard = new Dashboard(); + dashboard.setTitle("My dashboard"); + dashboard.setTenantId(savedTenant.getId()); + dashboard.setConfiguration(JacksonUtil.newObjectNode() + .set("widgets", JacksonUtil.toJsonNode(""" + {"xxx": + {"config":{"actions":{"elementClick":[ + {"customResources":[{"url":{"entityType":"TB_RESOURCE","id": + "tb-resource;/api/resource/js_module/tenant/gateway-management-extension.js"},"isModule":true}, + {"url":"tb-resource;/api/resource/js_module/tenant/gateway-management-extension.js","isModule":true}]}]}}}} + """)) + .put("someResource", link)); + + Dashboard savedDashboard = dashboardService.saveDashboard(dashboard); + Dashboard foundDashboard = dashboardService.findDashboardById(savedTenant.getId(), savedDashboard.getId()); + String resourceLink = foundDashboard.getConfiguration().get("someResource").asText(); + Assertions.assertNotNull(resourceLink); + Assert.assertEquals(resourceLink, link); + + TbResourceDeleteResult result = tbResourceService.delete(savedResource, true, null); + Assert.assertNotNull(result); + Assert.assertTrue(result.isSuccess()); + Assert.assertNull(result.getReferences()); + + foundResource = resourceService.findResourceById(savedTenant.getId(), savedResource.getId()); Assert.assertNull(foundResource); } diff --git a/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java index 39dbc7ca16..907e171985 100644 --- a/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.telemetry; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; -import org.checkerframework.checker.nullness.qual.NonNull; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -43,6 +42,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.objects.AttributesEntityView; import org.thingsboard.server.common.data.objects.TelemetryEntityView; @@ -56,6 +56,7 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldQueueService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.subscription.SubscriptionManagerService; @@ -73,6 +74,8 @@ import java.util.stream.Stream; import static com.google.common.util.concurrent.Futures.immediateFuture; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.BDDMockito.given; import static org.mockito.BDDMockito.then; import static org.mockito.Mockito.lenient; @@ -98,14 +101,6 @@ class DefaultTelemetrySubscriptionServiceTest { .myPartition(true) .build(); - final FutureCallback emptyCallback = new FutureCallback<>() { - @Override - public void onSuccess(Void result) {} - - @Override - public void onFailure(@NonNull Throwable t) {} - }; - ExecutorService wsCallBackExecutor; ExecutorService tsCallBackExecutor; @@ -125,12 +120,14 @@ class DefaultTelemetrySubscriptionServiceTest { TbApiUsageReportClient apiUsageClient; @Mock TbApiUsageStateService apiUsageStateService; + @Mock + CalculatedFieldQueueService calculatedFieldQueueService; DefaultTelemetrySubscriptionService telemetryService; @BeforeEach void setup() { - telemetryService = new DefaultTelemetrySubscriptionService(attrService, tsService, tbEntityViewService, apiUsageClient, apiUsageStateService); + telemetryService = new DefaultTelemetrySubscriptionService(attrService, tsService, tbEntityViewService, apiUsageClient, apiUsageStateService, calculatedFieldQueueService); ReflectionTestUtils.setField(telemetryService, "clusterService", clusterService); ReflectionTestUtils.setField(telemetryService, "partitionService", partitionService); ReflectionTestUtils.setField(telemetryService, "subscriptionManagerService", Optional.of(subscriptionManagerService)); @@ -147,13 +144,20 @@ class DefaultTelemetrySubscriptionServiceTest { lenient().when(partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId)).thenReturn(tpi); - lenient().when(tsService.save(tenantId, entityId, sampleTelemetry, sampleTtl)).thenReturn(immediateFuture(sampleTelemetry.size())); - lenient().when(tsService.saveWithoutLatest(tenantId, entityId, sampleTelemetry, sampleTtl)).thenReturn(immediateFuture(sampleTelemetry.size())); - lenient().when(tsService.saveLatest(tenantId, entityId, sampleTelemetry)).thenReturn(immediateFuture(listOfNNumbers(sampleTelemetry.size()))); + lenient().when(tsService.save(tenantId, entityId, sampleTelemetry, sampleTtl)).thenReturn(immediateFuture(TimeseriesSaveResult.of(sampleTelemetry.size(), listOfNNumbers(sampleTelemetry.size())))); + lenient().when(tsService.saveWithoutLatest(tenantId, entityId, sampleTelemetry, sampleTtl)).thenReturn(immediateFuture(TimeseriesSaveResult.of(sampleTelemetry.size(), null))); + lenient().when(tsService.saveLatest(tenantId, entityId, sampleTelemetry)).thenReturn(immediateFuture(TimeseriesSaveResult.of(sampleTelemetry.size(), listOfNNumbers(sampleTelemetry.size())))); // mock no entity views lenient().when(tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId)).thenReturn(immediateFuture(Collections.emptyList())); + // mock that calls to CF queue service are always successful + lenient().doAnswer(inv -> { + FutureCallback callback = inv.getArgument(2); + callback.onSuccess(null); + return null; + }).when(calculatedFieldQueueService).pushRequestToQueue(any(TimeseriesSaveRequest.class), any(), any()); + // send partition change event so currentPartitions set is populated telemetryService.onTbApplicationEvent(new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of(new QueueKey(ServiceType.TB_CORE), Set.of(tpi)), Collections.emptyMap())); } @@ -173,8 +177,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .strategy(new TimeseriesSaveRequest.Strategy(true, false, false)) - .callback(emptyCallback) + .strategy(new TimeseriesSaveRequest.Strategy(true, false, false, false)) .build(); // WHEN @@ -194,7 +197,6 @@ class DefaultTelemetrySubscriptionServiceTest { .entries(sampleTelemetry) .ttl(sampleTtl) .strategy(TimeseriesSaveRequest.Strategy.LATEST_AND_WS) - .callback(emptyCallback) .build(); // WHEN @@ -216,7 +218,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .strategy(TimeseriesSaveRequest.Strategy.SAVE_ALL) + .strategy(TimeseriesSaveRequest.Strategy.PROCESS_ALL) .future(future) .build(); @@ -265,7 +267,7 @@ class DefaultTelemetrySubscriptionServiceTest { // mock that there is one entity view given(tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId)).willReturn(immediateFuture(List.of(entityView))); // mock that save latest call for entity view is successful - given(tsService.saveLatest(tenantId, entityView.getId(), sampleTelemetry)).willReturn(immediateFuture(listOfNNumbers(sampleTelemetry.size()))); + given(tsService.saveLatest(tenantId, entityView.getId(), sampleTelemetry)).willReturn(immediateFuture(TimeseriesSaveResult.of(sampleTelemetry.size(), listOfNNumbers(sampleTelemetry.size())))); // mock TPI for entity view given(partitionService.resolve(ServiceType.TB_CORE, tenantId, entityView.getId())).willReturn(tpi); @@ -275,8 +277,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .strategy(new TimeseriesSaveRequest.Strategy(false, true, false)) - .callback(emptyCallback) + .strategy(new TimeseriesSaveRequest.Strategy(false, true, false, false)) .build(); // WHEN @@ -302,8 +303,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .strategy(new TimeseriesSaveRequest.Strategy(true, false, false)) - .callback(emptyCallback) + .strategy(new TimeseriesSaveRequest.Strategy(true, false, false, false)) .build(); // WHEN @@ -320,7 +320,7 @@ class DefaultTelemetrySubscriptionServiceTest { @ParameterizedTest @MethodSource("booleanCombinations") - void shouldCallCorrectApiBasedOnBooleanFlagsInTheSaveRequest(boolean saveTimeseries, boolean saveLatest, boolean sendWsUpdate) { + void shouldCallCorrectApiBasedOnBooleanFlagsInTheSaveRequest(boolean saveTimeseries, boolean saveLatest, boolean sendWsUpdate, boolean processCalculatedFields) { // GIVEN var request = TimeseriesSaveRequest.builder() .tenantId(tenantId) @@ -328,8 +328,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .strategy(new TimeseriesSaveRequest.Strategy(saveTimeseries, saveLatest, sendWsUpdate)) - .callback(emptyCallback) + .strategy(new TimeseriesSaveRequest.Strategy(saveTimeseries, saveLatest, sendWsUpdate, processCalculatedFields)) .build(); // WHEN @@ -343,6 +342,11 @@ class DefaultTelemetrySubscriptionServiceTest { } else if (saveTimeseries) { then(tsService).should().saveWithoutLatest(tenantId, entityId, sampleTelemetry, sampleTtl); } + + if (processCalculatedFields) { + then(calculatedFieldQueueService).should().pushRequestToQueue(eq(request), any(), eq(request.getCallback())); + } + then(tsService).shouldHaveNoMoreInteractions(); if (sendWsUpdate) { @@ -354,18 +358,26 @@ class DefaultTelemetrySubscriptionServiceTest { private static Stream booleanCombinations() { return Stream.of( - Arguments.of(true, true, true), - Arguments.of(true, true, false), - Arguments.of(true, false, true), - Arguments.of(true, false, false), - Arguments.of(false, true, true), - Arguments.of(false, true, false), - Arguments.of(false, false, true), - Arguments.of(false, false, false) + Arguments.of(true, true, true, true), + Arguments.of(true, true, true, false), + Arguments.of(true, true, false, true), + Arguments.of(true, true, false, false), + Arguments.of(true, false, true, true), + Arguments.of(true, false, true, false), + Arguments.of(true, false, false, true), + Arguments.of(true, false, false, false), + Arguments.of(false, true, true, true), + Arguments.of(false, true, true, false), + Arguments.of(false, true, false, true), + Arguments.of(false, true, false, false), + Arguments.of(false, false, true, true), + Arguments.of(false, false, true, false), + Arguments.of(false, false, false, true), + Arguments.of(false, false, false, false) ); } - // used to emulate sequence numbers returned by save latest API + // used to emulate versions returned by save latest API private static List listOfNNumbers(int N) { return LongStream.range(0, N).boxed().toList(); } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java index 52566f57f9..3193268bb2 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java @@ -45,7 +45,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @TestPropertySource(properties = { - "coap.enabled=true", + "coap.server.enabled=true", "service.integrations.supported=ALL", "transport.coap.enabled=true", }) diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/security/AbstractCoapSecurityIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/security/AbstractCoapSecurityIntegrationTest.java index 52e28cb5d0..72f8cc5b4f 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/security/AbstractCoapSecurityIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/security/AbstractCoapSecurityIntegrationTest.java @@ -61,7 +61,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @Slf4j @TestPropertySource(properties = { - "coap.enabled=true", + "coap.server.enabled=true", "coap.dtls.enabled=true", "coap.dtls.credentials.pem.cert_file=coap/credentials/server/cert.pem", "device.connectivity.coaps.enabled=true", diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index bf90333c52..3d0c377c2e 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -19,7 +19,7 @@ transport.mqtt.enabled=false transport.coap.enabled=false transport.lwm2m.enabled=false transport.snmp.enabled=false -coap.enabled=false +coap.server.enabled=false integrations.rpc.enabled=false service.integrations.supported=NONE @@ -35,6 +35,8 @@ sql.events.batch_threads=2 actors.system.tenant_dispatcher_pool_size=4 actors.system.device_dispatcher_pool_size=8 actors.system.rule_dispatcher_pool_size=12 +actors.system.cfm_dispatcher_pool_size=2 +actors.system.cfe_dispatcher_pool_size=2 transport.sessions.report_timeout=10000 queue.transport_api.request_poll_interval=5 queue.transport_api.response_poll_interval=5 diff --git a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java index 741908a14c..4b822e0030 100644 --- a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java +++ b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java @@ -153,7 +153,7 @@ public final class TbActorMailbox implements TbActorCtx { } if (msg != null) { try { - log.debug("[{}] Going to process message: {}", selfId, msg); + log.trace("[{}] Going to process message: {}", selfId, msg); actor.process(msg); } catch (TbRuleNodeUpdateException updateException) { stopReason = TbActorStopReason.INIT_FAILED; diff --git a/common/actor/src/main/java/org/thingsboard/server/actors/TbCalculatedFieldEntityActorId.java b/common/actor/src/main/java/org/thingsboard/server/actors/TbCalculatedFieldEntityActorId.java new file mode 100644 index 0000000000..3b69fe7eff --- /dev/null +++ b/common/actor/src/main/java/org/thingsboard/server/actors/TbCalculatedFieldEntityActorId.java @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors; + +import lombok.Getter; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.Objects; + +public class TbCalculatedFieldEntityActorId implements TbActorId { + + @Getter + private final EntityId entityId; + + public TbCalculatedFieldEntityActorId(EntityId entityId) { + this.entityId = entityId; + } + + @Override + public String toString() { + return entityId.getEntityType() + "|" + entityId.getId(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TbCalculatedFieldEntityActorId that = (TbCalculatedFieldEntityActorId) o; + return entityId.equals(that.entityId); + } + + @Override + public int hashCode() { + // Magic number to ensure that the hash does not match with the hash of other actor id - (TbEntityActorId) + return 42 + Objects.hash(entityId); + } + + @Override + public EntityType getEntityType() { + return entityId.getEntityType(); + } +} diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 1962e2f147..2a2d04e0fd 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -21,6 +21,8 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.EdgeId; @@ -36,7 +38,10 @@ import org.thingsboard.server.common.msg.edge.ToEdgeSyncRequest; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.RestApiCallResponseMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; @@ -58,6 +63,8 @@ public interface TbClusterService extends TbQueueClusterService { void broadcastToCore(ToCoreNotificationMsg msg); + void broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg build, TbQueueCallback callback); + void pushMsgToVersionControl(TenantId tenantId, ToVersionControlServiceMsg msg, TbQueueCallback callback); void pushNotificationToCore(String targetServiceId, FromDeviceRpcResponse response, TbQueueCallback callback); @@ -74,6 +81,12 @@ public interface TbClusterService extends TbQueueClusterService { void pushNotificationToTransport(String targetServiceId, ToTransportMsg response, TbQueueCallback callback); + void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, TransportProtos.ToCalculatedFieldMsg msg, TbQueueCallback callback); + + void pushMsgToCalculatedFields(TopicPartitionInfo tpi, UUID msgId, ToCalculatedFieldMsg msg, TbQueueCallback callback); + + void pushNotificationToCalculatedFields(TenantId tenantId, EntityId entityId, TransportProtos.ToCalculatedFieldNotificationMsg msg, TbQueueCallback callback); + void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); void onDeviceProfileChange(DeviceProfile deviceProfile, DeviceProfile oldDeviceProfile, TbQueueCallback callback); @@ -96,6 +109,10 @@ public interface TbClusterService extends TbQueueClusterService { void onDeviceAssignedToTenant(TenantId oldTenantId, Device device); + void onAssetUpdated(Asset asset, Asset old); + + void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback); + void onResourceChange(TbResourceInfo resource, TbQueueCallback callback); void onResourceDeleted(TbResourceInfo resource, TbQueueCallback callback); @@ -114,4 +131,8 @@ public interface TbClusterService extends TbQueueClusterService { void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId sourceEdgeId); + void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, TbQueueCallback callback); + + void onCalculatedFieldDeleted(CalculatedField calculatedField, TbQueueCallback callback); + } diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java index b8f9fa4e5f..e15d9c8ace 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java @@ -15,8 +15,22 @@ */ package org.thingsboard.server.queue; + public interface TbQueueCallback { + TbQueueCallback EMPTY = new TbQueueCallback() { + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + + } + + @Override + public void onFailure(Throwable t) { + + } + }; + void onSuccess(TbQueueMsgMetadata metadata); void onFailure(Throwable t); diff --git a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapServerComponent.java b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapServerComponent.java index e2d0f27184..98df973c65 100644 --- a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapServerComponent.java +++ b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapServerComponent.java @@ -21,6 +21,6 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${coap.enabled}'=='true')") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${coap.server.enabled}'=='true')") public @interface TbCoapServerComponent { } diff --git a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapTransportComponent.java b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapTransportComponent.java index 0c27c51a00..558ccf16ef 100644 --- a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapTransportComponent.java +++ b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapTransportComponent.java @@ -22,6 +22,6 @@ import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) @ConditionalOnExpression("'${service.type:null}'=='tb-transport' || " + - "('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${coap.enabled}'=='true' && '${transport.coap.enabled}'=='true')") + "('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${coap.server.enabled}'=='true' && '${transport.coap.enabled}'=='true')") public @interface TbCoapTransportComponent { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java index 04992800eb..09bc8f1f93 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.asset; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetSearchQuery; @@ -63,6 +64,10 @@ public interface AssetService extends EntityDaoService { PageData findAssetInfosByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink); + PageData findProfileEntityIdInfos(PageLink pageLink); + + PageData findAssetIdsByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink); + ListenableFuture> findAssetsByTenantIdAndIdsAsync(TenantId tenantId, List assetIds); void deleteAssetsByTenantId(TenantId tenantId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java new file mode 100644 index 0000000000..3d91790790 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.cf; + +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.entity.EntityDaoService; + +import java.util.List; + +public interface CalculatedFieldService extends EntityDaoService { + + CalculatedField save(CalculatedField calculatedField); + + CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId); + + List findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + + PageData findAllCalculatedFields(PageLink pageLink); + + PageData findAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink); + + void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + int deleteAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + + CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink); + + CalculatedFieldLink findCalculatedFieldLinkById(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId); + + List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + List findAllCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + + PageData findAllCalculatedFieldLinks(PageLink pageLink); + + boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); + +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java index bab59e6d66..4ef653855d 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.DeviceInfoFilter; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.device.DeviceSearchQuery; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; @@ -73,8 +74,12 @@ public interface DeviceService extends EntityDaoService { PageData findDeviceIdInfos(PageLink pageLink); + PageData findProfileEntityIdInfos(PageLink pageLink); + PageData findDevicesByTenantIdAndType(TenantId tenantId, String type, PageLink pageLink); + PageData findDeviceIdsByTenantIdAndDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId, PageLink pageLink); + PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, OtaPackageType type, PageLink pageLink); long countDevicesByTenantIdAndDeviceProfileIdAndEmptyOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, OtaPackageType otaPackageType); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java index d2fe51a133..65db0d5a76 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.entity; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.NameLabelAndCustomerDetails; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -34,6 +35,8 @@ public interface EntityService { Optional fetchEntityCustomerId(TenantId tenantId, EntityId entityId); + Optional> fetchEntity(TenantId tenantId, EntityId entityId); + Optional fetchNameLabelAndCustomerDetails(TenantId tenantId, EntityId entityId); long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java index d6074ca48b..ee187db46b 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.ResourceExportData; import org.thingsboard.server.common.data.ResourceSubType; import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TbResourceDeleteResult; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.TbResourceInfoFilter; import org.thingsboard.server.common.data.id.TbResourceId; @@ -69,9 +70,7 @@ public interface ResourceService extends EntityDaoService { PageData findTenantResourcesByResourceTypeAndPageLink(TenantId tenantId, ResourceType lwm2mModel, PageLink pageLink); - void deleteResource(TenantId tenantId, TbResourceId resourceId); - - void deleteResource(TenantId tenantId, TbResourceId resourceId, boolean force); + TbResourceDeleteResult deleteResource(TenantId tenantId, TbResourceId resourceId, boolean force); void deleteResourcesByTenantId(TenantId tenantId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java index 862c50e45e..e239e22ee9 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; @@ -44,13 +45,13 @@ public interface TimeseriesService { ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId); - ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); + ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); - ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); + ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); - ListenableFuture saveWithoutLatest(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); + ListenableFuture saveWithoutLatest(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); - ListenableFuture> saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntry); + ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntries); ListenableFuture> remove(TenantId tenantId, EntityId entityId, List queries); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java index 98bf6363b8..ad02ade403 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java @@ -34,4 +34,5 @@ public interface ApiUsageStateService extends EntityDaoService { void deleteApiUsageStateByEntityId(EntityId entityId); ApiUsageState findApiUsageStateById(TenantId tenantId, ApiUsageStateId id); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java index d3f57e6efa..5a102f31e2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ApiUsageState.java @@ -27,7 +27,7 @@ import org.thingsboard.server.common.data.id.TenantId; @EqualsAndHashCode(callSuper = true) @Getter @Setter -public class ApiUsageState extends BaseData implements HasTenantId { +public class ApiUsageState extends BaseData implements HasTenantId, HasVersion { private static final long serialVersionUID = 8250339805336035966L; @@ -41,6 +41,7 @@ public class ApiUsageState extends BaseData implements HasTenan private ApiUsageStateValue emailExecState; private ApiUsageStateValue smsExecState; private ApiUsageStateValue alarmExecState; + private Long version; public ApiUsageState() { super(); @@ -62,6 +63,7 @@ public class ApiUsageState extends BaseData implements HasTenan this.emailExecState = ur.getEmailExecState(); this.smsExecState = ur.getSmsExecState(); this.alarmExecState = ur.getAlarmExecState(); + this.version = ur.getVersion(); } public boolean isTransportEnabled() { @@ -84,15 +86,16 @@ public class ApiUsageState extends BaseData implements HasTenan return !ApiUsageStateValue.DISABLED.equals(tbelExecState); } - public boolean isEmailSendEnabled(){ + public boolean isEmailSendEnabled() { return !ApiUsageStateValue.DISABLED.equals(emailExecState); } - public boolean isSmsSendEnabled(){ + public boolean isSmsSendEnabled() { return !ApiUsageStateValue.DISABLED.equals(smsExecState); } public boolean isAlarmCreationEnabled() { return alarmExecState != ApiUsageStateValue.DISABLED; } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java index e80a5d86e7..b53d6daec2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java @@ -145,4 +145,7 @@ public class DataConstants { public static final String EDGE_QUEUE_NAME = "Edge"; public static final String EDGE_EVENT_QUEUE_NAME = "EdgeEvent"; + public static final String CF_QUEUE_NAME = "CalculatedFields"; + public static final String CF_STATES_QUEUE_NAME = "CalculatedFieldStates"; + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index f5de438a9f..4ea66be1b4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -61,7 +61,9 @@ public enum EntityType { OAUTH2_CLIENT(35), DOMAIN(36), MOBILE_APP(37), - MOBILE_APP_BUNDLE(38); + MOBILE_APP_BUNDLE(38), + CALCULATED_FIELD(39), + CALCULATED_FIELD_LINK(40); @Getter private final int protoNumber; // Corresponds to EntityTypeProto diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java new file mode 100644 index 0000000000..d1b4a35b7b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.io.Serializable; +import java.util.UUID; + +@Data +@Slf4j +public class ProfileEntityIdInfo implements Serializable, HasTenantId { + + private static final long serialVersionUID = 8532058281983868003L; + + private final TenantId tenantId; + private final EntityId profileId; + private final EntityId entityId; + + private ProfileEntityIdInfo(UUID tenantId, EntityId profileId, EntityId entityId) { + this.tenantId = TenantId.fromUUID(tenantId); + this.profileId = profileId; + this.entityId = entityId; + } + + public static ProfileEntityIdInfo create(UUID tenantId, DeviceProfileId profileId, DeviceId entityId) { + return new ProfileEntityIdInfo(tenantId, profileId, entityId); + } + + public static ProfileEntityIdInfo create(UUID tenantId, AssetProfileId profileId, AssetId entityId) { + return new ProfileEntityIdInfo(tenantId, profileId, entityId); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java index 0856f65264..b1ef4d7f22 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java @@ -34,4 +34,7 @@ public class SystemParams { boolean mobileQrEnabled; int maxDebugModeDurationMinutes; String ruleChainDebugPerTenantLimitsConfiguration; + String calculatedFieldDebugPerTenantLimitsConfiguration; + long maxArgumentsPerCF; + long maxDataPointsPerRollingArg; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TbResourceDeleteResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/TbResourceDeleteResult.java new file mode 100644 index 0000000000..edc5a2f539 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/TbResourceDeleteResult.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.id.HasId; + +import java.util.List; +import java.util.Map; + +@Data +@Builder +public class TbResourceDeleteResult { + + private boolean success; + private Map>> references; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java new file mode 100644 index 0000000000..d4aa0b60a3 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -0,0 +1,133 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSetter; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasDebugSettings; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.HasVersion; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.validation.Length; +import org.thingsboard.server.common.data.validation.NoXss; + +@Schema +@Data +@EqualsAndHashCode(callSuper = true) +public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion, ExportableEntity, HasDebugSettings { + + private static final long serialVersionUID = 4491966747773381420L; + + private TenantId tenantId; + private EntityId entityId; + + @NoXss + @Length(fieldName = "type") + private CalculatedFieldType type; + @NoXss + @Length(fieldName = "name") + @Schema(description = "User defined name of the calculated field.") + private String name; + @Deprecated + @Schema(description = "Enable/disable debug. ", example = "false", deprecated = true) + private boolean debugMode; + @Schema(description = "Debug settings object.") + private DebugSettings debugSettings; + @Schema(description = "Version of calculated field configuration.", example = "0") + private int configurationVersion; + @Schema(implementation = SimpleCalculatedFieldConfiguration.class) + private transient CalculatedFieldConfiguration configuration; + @Getter + @Setter + private Long version; + @Getter + @Setter + private CalculatedFieldId externalId; + + public CalculatedField() { + super(); + } + + public CalculatedField(CalculatedFieldId id) { + super(id); + } + + public CalculatedField(TenantId tenantId, EntityId entityId, CalculatedFieldType type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version, CalculatedFieldId externalId) { + this.tenantId = tenantId; + this.entityId = entityId; + this.type = type; + this.name = name; + this.configurationVersion = configurationVersion; + this.configuration = configuration; + this.version = version; + this.externalId = externalId; + } + + @Schema(description = "JSON object with the Calculated Field Id. Referencing non-existing Calculated Field Id will cause error.") + @Override + public CalculatedFieldId getId() { + return super.getId(); + } + + @Schema(description = "Timestamp of the calculated field creation, in milliseconds", example = "1609459200000", accessMode = Schema.AccessMode.READ_ONLY) + @Override + public long getCreatedTime() { + return super.getCreatedTime(); + } + + @Override + public String toString() { + return new StringBuilder() + .append("CalculatedField[") + .append("tenantId=").append(tenantId) + .append(", entityId=").append(entityId) + .append(", type='").append(type) + .append(", name='").append(name) + .append(", configurationVersion=").append(configurationVersion) + .append(", configuration=").append(configuration) + .append(", version=").append(version) + .append(", externalId=").append(externalId) + .append(", createdTime=").append(createdTime) + .append(", id=").append(id).append(']') + .toString(); + } + + // Getter is ignored for serialization + @JsonIgnore + public boolean isDebugMode() { + return debugMode; + } + + // Setter is annotated for deserialization + @JsonSetter + public void setDebugMode(boolean debugMode) { + this.debugMode = debugMode; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java new file mode 100644 index 0000000000..3f048815da --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java @@ -0,0 +1,66 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +@Schema +@Data +@EqualsAndHashCode(callSuper = true) +public class CalculatedFieldLink extends BaseData { + + private static final long serialVersionUID = 6492846246722091530L; + + private TenantId tenantId; + private EntityId entityId; + + @Schema(description = "JSON object with the Calculated Field Id. ", accessMode = Schema.AccessMode.READ_ONLY) + private CalculatedFieldId calculatedFieldId; + + public CalculatedFieldLink() { + super(); + } + + public CalculatedFieldLink(CalculatedFieldLinkId id) { + super(id); + } + + public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId) { + this.tenantId = tenantId; + this.entityId = entityId; + this.calculatedFieldId = calculatedFieldId; + } + + @Override + public String toString() { + return new StringBuilder() + .append("CalculatedFieldLink[") + .append("tenantId=").append(tenantId) + .append(", entityId=").append(entityId) + .append(", calculatedFieldId=").append(calculatedFieldId) + .append(", createdTime=").append(createdTime) + .append(", id=").append(id).append(']') + .toString(); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java new file mode 100644 index 0000000000..acef67a041 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +public enum CalculatedFieldType { + + SIMPLE, SCRIPT + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java new file mode 100644 index 0000000000..e7daa70b1b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Data; +import org.springframework.lang.Nullable; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Argument { + + @Nullable + private EntityId refEntityId; + private ReferencedEntityKey refEntityKey; + private String defaultValue; + + private Integer limit; + private Long timeWindow; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java new file mode 100644 index 0000000000..7f057f4038 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +public enum ArgumentType { + + TS_LATEST, ATTRIBUTE, TS_ROLLING + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..8227ff4603 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +@Data +public abstract class BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + protected Map arguments; + protected String expression; + protected Output output; + + @Override + public List getReferencedEntities() { + return arguments.values().stream() + .map(Argument::getRefEntityId) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + @Override + public List buildCalculatedFieldLinks(TenantId tenantId, EntityId cfEntityId, CalculatedFieldId calculatedFieldId) { + return getReferencedEntities().stream() + .filter(referencedEntity -> !referencedEntity.equals(cfEntityId)) + .map(referencedEntityId -> buildCalculatedFieldLink(tenantId, referencedEntityId, calculatedFieldId)) + .collect(Collectors.toList()); + } + + @Override + public CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, EntityId referencedEntityId, CalculatedFieldId calculatedFieldId) { + CalculatedFieldLink link = new CalculatedFieldLink(); + link.setTenantId(tenantId); + link.setEntityId(referencedEntityId); + link.setCalculatedFieldId(calculatedFieldId); + return link; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java new file mode 100644 index 0000000000..c53f1fe5f1 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.List; +import java.util.Map; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE"), + @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT") +}) +public interface CalculatedFieldConfiguration { + + @JsonIgnore + CalculatedFieldType getType(); + + Map getArguments(); + + String getExpression(); + + void setExpression(String expression); + + Output getOutput(); + + @JsonIgnore + List getReferencedEntities(); + + List buildCalculatedFieldLinks(TenantId tenantId, EntityId cfEntityId, CalculatedFieldId calculatedFieldId); + + CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, EntityId referencedEntityId, CalculatedFieldId calculatedFieldId); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java new file mode 100644 index 0000000000..49e393b19f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; + +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Output { + + private String name; + private OutputType type; + private AttributeScope scope; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java new file mode 100644 index 0000000000..04a816b74f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +public enum OutputType { + + TIME_SERIES, ATTRIBUTES + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java new file mode 100644 index 0000000000..9e3a75c891 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; + +@Data +@AllArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ReferencedEntityKey { + + private String key; + private ArgumentType type; + private AttributeScope scope; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..0971217fdf --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; + +@Data +public class ScriptCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SCRIPT; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..79a0518ba0 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; + +@Data +public class SimpleCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SIMPLE; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java index 3524fdbcbf..d10f375bc1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java @@ -43,8 +43,9 @@ public class ApiUsageStateFields extends AbstractEntityFields { public ApiUsageStateFields(UUID id, long createdTime, UUID tenantId, UUID entityId, String entityType, ApiUsageStateValue transportState, ApiUsageStateValue dbStorageState, ApiUsageStateValue reExecState, ApiUsageStateValue jsExecState, ApiUsageStateValue tbelExecState, - ApiUsageStateValue emailExecState, ApiUsageStateValue smsExecState, ApiUsageStateValue alarmExecState) { - super(id, createdTime, tenantId); + ApiUsageStateValue emailExecState, ApiUsageStateValue smsExecState, ApiUsageStateValue alarmExecState, + Long version) { + super(id, createdTime, tenantId, null, null, version); this.entityId = (entityType != null && entityId != null) ? EntityIdFactory.getByTypeAndUuid(entityType, entityId) : null; this.transportState = transportState; this.dbStorageState = dbStorageState; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java index 1bbf8a519a..532c4a92ac 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.edqs.fields; +import com.fasterxml.jackson.annotation.JsonInclude; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.thingsboard.server.common.data.id.EntityId; @@ -23,6 +24,7 @@ import java.util.Collections; import java.util.List; import java.util.UUID; +@JsonInclude(JsonInclude.Include.NON_NULL) public interface EntityFields { Logger log = LoggerFactory.getLogger(EntityFields.class); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java index 281a4bdccf..a36514248c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java @@ -284,6 +284,7 @@ public class FieldsUtil { .emailExecState(entity.getEmailExecState()) .smsExecState(entity.getSmsExecState()) .alarmExecState(entity.getAlarmExecState()) + .version(entity.getVersion()) .build(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java new file mode 100644 index 0000000000..0424eabeb6 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java @@ -0,0 +1,94 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; + +@ToString +@EqualsAndHashCode(callSuper = true) +public class CalculatedFieldDebugEvent extends Event { + + private static final long serialVersionUID = -7091690784759639853L; + + @Builder + private CalculatedFieldDebugEvent(TenantId tenantId, UUID entityId, String serviceId, UUID id, long ts, + CalculatedFieldId calculatedFieldId, EntityId eventEntity, UUID msgId, + String msgType, String arguments, String result, String error) { + super(tenantId, entityId, serviceId, id, ts); + this.calculatedFieldId = calculatedFieldId; + this.eventEntity = eventEntity; + this.msgId = msgId; + this.msgType = msgType; + this.arguments = arguments; + this.result = result; + this.error = error; + } + + @Getter + private final CalculatedFieldId calculatedFieldId; + @Getter + private final EntityId eventEntity; + @Getter + private final UUID msgId; + @Getter + private final String msgType; + @Getter + @Setter + private String arguments; + @Getter + @Setter + private String result; + @Getter + @Setter + private String error; + + @Override + public EventType getType() { + return EventType.DEBUG_CALCULATED_FIELD; + } + + @Override + public EventInfo toInfo(EntityType entityType) { + EventInfo eventInfo = super.toInfo(entityType); + var json = (ObjectNode) eventInfo.getBody(); + json.put("calculatedFieldId", calculatedFieldId.toString()); + if (eventEntity != null) { + json.put("entityId", eventEntity.getId().toString()) + .put("entityType", eventEntity.getEntityType().name()); + } + if (msgId != null) { + json.put("msgId", msgId.toString()); + } + putNotNull(json, "msgType", msgType); + putNotNull(json, "arguments", arguments); + putNotNull(json, "result", result); + putNotNull(json, "error", error); + return eventInfo; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java new file mode 100644 index 0000000000..55ce036d9e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.StringUtils; + +@Data +@EqualsAndHashCode(callSuper = true) +@Schema +public class CalculatedFieldDebugEventFilter extends DebugEventFilter { + + @Schema(description = "String value representing the entity id in the event body", example = "57b6bafe-d600-423c-9267-fe31e5218986") + protected String entityId; + @Schema(description = "String value representing the entity type", allowableValues = "DEVICE") + protected String entityType; + @Schema(description = "String value representing the message id in the rule engine", example = "dcf44612-2ce4-4e5d-b462-ebb9c5628228") + protected String msgId; + @Schema(description = "String value representing the message type", example = "POST_TELEMETRY_REQUEST") + protected String msgType; + @Schema(description = "String value representing the arguments that were used in the calculation performed", + example = "{\"x\":{\"ts\":1739432016629,\"value\":20},\"y\":{\"ts\":1739429717656,\"value\":12}}") + protected String arguments; + @Schema(description = "String value representing the result of a calculation", + example = "{\"x + y\":54}") + protected String result; + + + @Override + public EventType getEventType() { + return EventType.DEBUG_CALCULATED_FIELD; + } + + @Override + public boolean isNotEmpty() { + return super.isNotEmpty() || !StringUtils.isEmpty(entityId) || !StringUtils.isEmpty(entityType) + || !StringUtils.isEmpty(msgId) || !StringUtils.isEmpty(msgType) + || !StringUtils.isEmpty(arguments) || !StringUtils.isEmpty(result); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java index 454d04f490..748771d1eb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java @@ -29,7 +29,8 @@ import io.swagger.v3.oas.annotations.media.Schema; @JsonSubTypes.Type(value = RuleChainDebugEventFilter.class, name = "DEBUG_RULE_CHAIN"), @JsonSubTypes.Type(value = ErrorEventFilter.class, name = "ERROR"), @JsonSubTypes.Type(value = LifeCycleEventFilter.class, name = "LC_EVENT"), - @JsonSubTypes.Type(value = StatisticsEventFilter.class, name = "STATS") + @JsonSubTypes.Type(value = StatisticsEventFilter.class, name = "STATS"), + @JsonSubTypes.Type(value = CalculatedFieldDebugEventFilter.class, name = "DEBUG_CALCULATED_FIELD") }) public interface EventFilter { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java index af75a92ea6..ce529c81bc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java @@ -22,7 +22,8 @@ public enum EventType { LC_EVENT("lc_event", "LC_EVENT"), STATS("stats_event", "STATS"), DEBUG_RULE_NODE("rule_node_debug_event", "DEBUG_RULE_NODE", true), - DEBUG_RULE_CHAIN("rule_chain_debug_event", "DEBUG_RULE_CHAIN", true); + DEBUG_RULE_CHAIN("rule_chain_debug_event", "DEBUG_RULE_CHAIN", true), + DEBUG_CALCULATED_FIELD("cf_debug_event", "DEBUG_CALCULATED_FIELD", true); @Getter private final String table; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java index caeef6bd2f..ed02d22a74 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java @@ -81,6 +81,10 @@ public class HousekeeperTask implements Serializable { return new TenantEntitiesDeletionHousekeeperTask(tenantId, entityType); } + public static HousekeeperTask deleteCalculatedFields(TenantId tenantId, EntityId entityId) { + return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_CALCULATED_FIELDS); + } + @JsonIgnore public String getDescription() { return taskType.getDescription() + " for " + entityId.getEntityType().getNormalName().toLowerCase() + " " + entityId.getId(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java index 1331b175ac..ef217debc3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java @@ -30,7 +30,8 @@ public enum HousekeeperTaskType { DELETE_ALARMS("alarms deletion"), UNASSIGN_ALARMS("alarms unassigning"), DELETE_TENANT_ENTITIES("tenant entities deletion"), - DELETE_ENTITIES("entities deletion"); + DELETE_ENTITIES("entities deletion"), + DELETE_CALCULATED_FIELDS("calculated fields deletion"); private final String description; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java new file mode 100644 index 0000000000..0a83f1a19f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.id; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import org.thingsboard.server.common.data.EntityType; + +import java.util.UUID; + +@Schema +public class CalculatedFieldId extends UUIDBased implements EntityId { + + private static final long serialVersionUID = 1L; + + @JsonCreator + public CalculatedFieldId(@JsonProperty("id") UUID id) { + super(id); + } + + public static CalculatedFieldId fromString(String calculatedFieldId) { + return new CalculatedFieldId(UUID.fromString(calculatedFieldId)); + } + + @Schema(requiredMode = Schema.RequiredMode.REQUIRED, description = "string", example = "CALCULATED_FIELD", allowableValues = "CALCULATED_FIELD") + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldLinkId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldLinkId.java new file mode 100644 index 0000000000..6a0c680bb6 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldLinkId.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.id; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import org.thingsboard.server.common.data.EntityType; + +import java.util.UUID; + +@Schema +public class CalculatedFieldLinkId extends UUIDBased implements EntityId { + + private static final long serialVersionUID = 1L; + + @JsonCreator + public CalculatedFieldLinkId(@JsonProperty("id") UUID id) { + super(id); + } + + public static CalculatedFieldLinkId fromString(String calculatedFieldLinkId) { + return new CalculatedFieldLinkId(UUID.fromString(calculatedFieldLinkId)); + } + + @Schema(requiredMode = Schema.RequiredMode.REQUIRED, description = "string", example = "CALCULATED_FIELD_LINK", allowableValues = "CALCULATED_FIELD_LINK") + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD_LINK; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index 7295f9795a..f5dd4b12a0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -113,6 +113,10 @@ public class EntityIdFactory { return new DomainId(uuid); case MOBILE_APP_BUNDLE: return new MobileAppBundleId(uuid); + case CALCULATED_FIELD: + return new CalculatedFieldId(uuid); + case CALCULATED_FIELD_LINK: + return new CalculatedFieldLinkId(uuid); } throw new IllegalArgumentException("EntityType " + type + " is not supported!"); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TimeseriesSaveResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TimeseriesSaveResult.java new file mode 100644 index 0000000000..233eb8df73 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TimeseriesSaveResult.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.kv; + +import lombok.Data; + +import java.util.List; + +@Data(staticConstructor = "of") +public class TimeseriesSaveResult { + + public static final TimeseriesSaveResult EMPTY = new TimeseriesSaveResult(0, null); + + private final Integer dataPoints; + private final List versions; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java b/common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java index a25d12577f..db7f14171b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java @@ -43,7 +43,8 @@ public enum LimitedApi { TRANSPORT_MESSAGES_PER_GATEWAY("transport messages per gateway", false), TRANSPORT_MESSAGES_PER_GATEWAY_DEVICE("transport messages per gateway device", false), EMAILS("emails sending", true), - WS_SUBSCRIPTIONS("WS subscriptions", false); + WS_SUBSCRIPTIONS("WS subscriptions", false), + CALCULATED_FIELD_DEBUG_EVENTS("calculated field debug events", true); private Function configExtractor; @Getter diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/queue/ProcessingStrategyType.java b/common/data/src/main/java/org/thingsboard/server/common/data/queue/ProcessingStrategyType.java index 5b8c86d7d1..ca63d34a84 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/queue/ProcessingStrategyType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/queue/ProcessingStrategyType.java @@ -13,21 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.thingsboard.server.common.data.queue; public enum ProcessingStrategyType { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index f62b3aaa9f..9be8a126b2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -135,6 +135,12 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private double warnThreshold; + private long maxCalculatedFieldsPerEntity; + private long maxArgumentsPerCF; + private long maxDataPointsPerRollingArg; + private long maxStateSizeInKBytes; + private long maxSingleValueArgumentSizeInKBytes; + @Override public long getProfileThreshold(ApiUsageRecordKey key) { return switch (key) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java index 0ba4c2125e..71c5256203 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java @@ -83,4 +83,16 @@ public class CollectionsUtil { return result; } + public static boolean isOneOf(V value, V... others) { + if (value == null) { + return false; + } + for (V other : others) { + if (value.equals(other)) { + return true; + } + } + return false; + } + } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java index 5020ed3476..0c55bc50dd 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java @@ -219,7 +219,9 @@ public class TenantRepo { EntityType entityType = entity.getType(); EntityData removed = getEntityMap(entityType).remove(entityId); if (removed != null) { - getEntitySet(entityType).remove(removed); + if (removed.getFields() != null) { + getEntitySet(entityType).remove(removed); + } edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.DELETED)); UUID customerId = removed.getCustomerId(); if (customerId != null) { diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java index 781ae2ed3d..5b4cd7ac4a 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java @@ -15,12 +15,15 @@ */ package org.thingsboard.server.edqs.util; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; import com.google.protobuf.ByteString; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ObjectType; @@ -217,16 +220,24 @@ public class EdqsConverter { @RequiredArgsConstructor private static class JsonConverter implements Converter { + private static final ObjectMapper mapper = JsonMapper.builder() + .visibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY) + .visibility(PropertyAccessor.GETTER, JsonAutoDetect.Visibility.NONE) + .visibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE) + .build(); + private final Class type; + @SneakyThrows @Override public byte[] serialize(ObjectType objectType, T value) { - return JacksonUtil.writeValueAsBytes(value); + return mapper.writeValueAsBytes(value); } + @SneakyThrows @Override public T deserialize(ObjectType objectType, byte[] bytes) { - return JacksonUtil.fromBytes(bytes, this.type); + return mapper.readValue(bytes, this.type); } } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java index ae1436e6ca..4a991432c7 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java @@ -19,6 +19,7 @@ import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Getter; import org.rocksdb.Options; +import org.rocksdb.WriteOptions; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.thingsboard.server.queue.edqs.InMemoryEdqsComponent; @@ -34,16 +35,18 @@ public class EdqsRocksDb extends TbRocksDb { private boolean isNew; public EdqsRocksDb(@Value("${queue.edqs.local.rocksdb_path:${user.home}/.rocksdb/edqs}") String path) { - super(path, new Options().setCreateIfMissing(true)); + super(path, new Options().setCreateIfMissing(true), new WriteOptions()); } @PostConstruct + @Override public void init() { isNew = !Files.exists(Path.of(path)); super.init(); } @PreDestroy + @Override public void close() { super.close(); } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbRocksDb.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbRocksDb.java index cd5ba0c9d9..23f2fa2c9e 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbRocksDb.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbRocksDb.java @@ -15,35 +15,43 @@ */ package org.thingsboard.server.edqs.util; -import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import org.rocksdb.Options; import org.rocksdb.RocksDB; -import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; +import org.rocksdb.WriteOptions; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.function.BiConsumer; -@RequiredArgsConstructor public class TbRocksDb { protected final String path; - private final Options options; - - private RocksDB db; + private final Options dbOptions; + private final WriteOptions writeOptions; + protected RocksDB db; static { RocksDB.loadLibrary(); } + public TbRocksDb(String path, Options dbOptions, WriteOptions writeOptions) { + this.path = path; + this.dbOptions = dbOptions; + this.writeOptions = writeOptions; + } + @SneakyThrows public void init() { - db = RocksDB.open(options, path); + Files.createDirectories(Path.of(path).getParent()); + db = RocksDB.open(dbOptions, path); } - public void put(String key, byte[] value) throws RocksDBException { - db.put(key.getBytes(StandardCharsets.UTF_8), value); + @SneakyThrows + public void put(String key, byte[] value) { + db.put(writeOptions, key.getBytes(StandardCharsets.UTF_8), value); } public void forEach(BiConsumer processor) { @@ -55,8 +63,9 @@ public class TbRocksDb { } } - public void delete(String key) throws RocksDBException { - db.delete(key.getBytes(StandardCharsets.UTF_8)); + @SneakyThrows + public void delete(String key) { + db.delete(writeOptions, key.getBytes(StandardCharsets.UTF_8)); } public void close() { diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index d5a6ff99b2..178caf7961 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -133,7 +133,22 @@ public enum MsgType { * Messages that are sent to and from edge session to start edge synchronization process */ EDGE_SYNC_REQUEST_TO_EDGE_SESSION_MSG, - EDGE_SYNC_RESPONSE_FROM_EDGE_SESSION_MSG; + EDGE_SYNC_RESPONSE_FROM_EDGE_SESSION_MSG, + + + CF_INIT_MSG, // Sent to init particular calculated field; + CF_LINK_INIT_MSG, // Sent to init particular calculated field; + CF_STATE_RESTORE_MSG, // Sent to restore particular calculated field entity state; + CF_PARTITIONS_CHANGE_MSG, // Sent when cluster event occures; + + CF_ENTITY_LIFECYCLE_MSG, // Sent on CF/Device/Asset create/update/delete; + CF_TELEMETRY_MSG, // Sent from queue to actor system; + CF_LINKED_TELEMETRY_MSG, // Sent from queue to actor system; + + /* CF Manager Actor -> CF Entity actor */ + CF_ENTITY_TELEMETRY_MSG, + CF_ENTITY_INIT_CF_MSG, + CF_ENTITY_DELETE_MSG; @Getter private final boolean ignoreOnStart; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 5592244bd9..f1c2236080 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -24,6 +24,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -34,8 +35,10 @@ import org.thingsboard.server.common.msg.gen.MsgProtos; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.io.Serializable; +import java.util.List; import java.util.Objects; import java.util.UUID; +import java.util.concurrent.CopyOnWriteArrayList; /** * Created by ashvayka on 13.01.18. @@ -64,6 +67,8 @@ public final class TbMsg implements Serializable { private final UUID correlationId; private final Integer partition; + private final List previousCalculatedFieldIds; + @Getter(value = AccessLevel.NONE) @JsonIgnore //This field is not serialized because we use queues and there is no need to do it @@ -112,7 +117,7 @@ public final class TbMsg implements Serializable { } private TbMsg(String queueName, UUID id, long ts, TbMsgType internalType, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, UUID correlationId, Integer partition, TbMsgProcessingCtx ctx, TbMsgCallback callback) { + RuleChainId ruleChainId, RuleNodeId ruleNodeId, UUID correlationId, Integer partition, List previousCalculatedFieldIds, TbMsgProcessingCtx ctx, TbMsgCallback callback) { this.id = id != null ? id : UUID.randomUUID(); this.queueName = queueName; if (ts > 0) { @@ -139,6 +144,9 @@ public final class TbMsg implements Serializable { this.ruleNodeId = ruleNodeId; this.correlationId = correlationId; this.partition = partition; + this.previousCalculatedFieldIds = previousCalculatedFieldIds != null + ? new CopyOnWriteArrayList<>(previousCalculatedFieldIds) + : new CopyOnWriteArrayList<>(); this.ctx = ctx != null ? ctx : new TbMsgProcessingCtx(); this.callback = Objects.requireNonNullElse(callback, TbMsgCallback.EMPTY); } @@ -186,6 +194,16 @@ public final class TbMsg implements Serializable { builder.setPartition(msg.getPartition()); } + if (msg.getPreviousCalculatedFieldIds() != null) { + for (CalculatedFieldId calculatedFieldId : msg.getPreviousCalculatedFieldIds()) { + MsgProtos.CalculatedFieldIdProto calculatedFieldIdProto = MsgProtos.CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) + .build(); + builder.addCalculatedFields(calculatedFieldIdProto); + } + } + builder.setCtx(msg.ctx.toProto()); return builder.build().toByteArray(); } @@ -200,6 +218,7 @@ public final class TbMsg implements Serializable { RuleNodeId ruleNodeId = null; UUID correlationId = null; Integer partition = null; + List calculatedFieldIds = new CopyOnWriteArrayList<>(); if (proto.getCustomerIdMSB() != 0L && proto.getCustomerIdLSB() != 0L) { customerId = new CustomerId(new UUID(proto.getCustomerIdMSB(), proto.getCustomerIdLSB())); } @@ -214,6 +233,14 @@ public final class TbMsg implements Serializable { partition = proto.getPartition(); } + for (MsgProtos.CalculatedFieldIdProto cfIdProto : proto.getCalculatedFieldsList()) { + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID( + cfIdProto.getCalculatedFieldIdMSB(), + cfIdProto.getCalculatedFieldIdLSB() + )); + calculatedFieldIds.add(calculatedFieldId); + } + TbMsgProcessingCtx ctx; if (proto.hasCtx()) { ctx = TbMsgProcessingCtx.fromProto(proto.getCtx()); @@ -224,7 +251,7 @@ public final class TbMsg implements Serializable { TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), null, proto.getType(), entityId, customerId, - metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, correlationId, partition, ctx, callback); + metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, correlationId, partition, calculatedFieldIds, ctx, callback); } catch (InvalidProtocolBufferException e) { throw new IllegalStateException("Could not parse protobuf for TbMsg", e); } @@ -249,6 +276,7 @@ public final class TbMsg implements Serializable { /** * Checks if the message is still valid for processing. May be invalid if the message pack is timed-out or canceled. + * * @return 'true' if message is valid for processing, 'false' otherwise. */ public boolean isValid() { @@ -343,10 +371,12 @@ public final class TbMsg implements Serializable { protected RuleNodeId ruleNodeId; protected UUID correlationId; protected Integer partition; + protected List previousCalculatedFieldIds; protected TbMsgProcessingCtx ctx; protected TbMsgCallback callback; - TbMsgBuilder() {} + TbMsgBuilder() { + } TbMsgBuilder(TbMsg tbMsg) { this.queueName = tbMsg.queueName; @@ -363,6 +393,7 @@ public final class TbMsg implements Serializable { this.ruleNodeId = tbMsg.ruleNodeId; this.correlationId = tbMsg.correlationId; this.partition = tbMsg.partition; + this.previousCalculatedFieldIds = tbMsg.previousCalculatedFieldIds; this.ctx = tbMsg.ctx; this.callback = tbMsg.callback; } @@ -385,8 +416,7 @@ public final class TbMsg implements Serializable { /** *

Deprecated: This should only be used when you need to specify a custom message type that doesn't exist in the {@link TbMsgType} enum. * Prefer using {@link #type(TbMsgType)} instead. - * - * */ + */ @Deprecated public TbMsgBuilder type(String type) { this.type = type; @@ -454,6 +484,11 @@ public final class TbMsg implements Serializable { return this; } + public TbMsgBuilder previousCalculatedFieldIds(List previousCalculatedFieldIds) { + this.previousCalculatedFieldIds = new CopyOnWriteArrayList<>(previousCalculatedFieldIds); + return this; + } + public TbMsgBuilder ctx(TbMsgProcessingCtx ctx) { this.ctx = ctx; return this; @@ -465,7 +500,7 @@ public final class TbMsg implements Serializable { } public TbMsg build() { - return new TbMsg(queueName, id, ts, internalType, type, originator, customerId, metaData, dataType, data, ruleChainId, ruleNodeId, correlationId, partition, ctx, callback); + return new TbMsg(queueName, id, ts, internalType, type, originator, customerId, metaData, dataType, data, ruleChainId, ruleNodeId, correlationId, partition, previousCalculatedFieldIds, ctx, callback); } public String toString() { @@ -473,8 +508,8 @@ public final class TbMsg implements Serializable { ", type=" + this.type + ", internalType=" + this.internalType + ", originator=" + this.originator + ", customerId=" + this.customerId + ", metaData=" + this.metaData + ", dataType=" + this.dataType + ", data=" + this.data + ", ruleChainId=" + this.ruleChainId + ", ruleNodeId=" + this.ruleNodeId + - ", correlationId=" + this.correlationId + ", partition=" + this.partition + ", ctx=" + this.ctx + - ", callback=" + this.callback + ")"; + ", correlationId=" + this.correlationId + ", partition=" + this.partition + ", previousCalculatedFields=" + this.previousCalculatedFieldIds + + ", ctx=" + this.ctx + ", callback=" + this.callback + ")"; } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/ToCalculatedFieldSystemMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/ToCalculatedFieldSystemMsg.java new file mode 100644 index 0000000000..c05c0f121e --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/ToCalculatedFieldSystemMsg.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg; + +import org.thingsboard.server.common.msg.aware.TenantAwareMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; + +public interface ToCalculatedFieldSystemMsg extends TenantAwareMsg { + + default TbCallback getCallback() { + return TbCallback.EMPTY; + } + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldEntityLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldEntityLifecycleMsg.java new file mode 100644 index 0000000000..099240f54d --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldEntityLifecycleMsg.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; + +@Data +public class CalculatedFieldEntityLifecycleMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final ComponentLifecycleMsg data; + private final TbCallback callback; + + @Override + public MsgType getMsgType() { + return MsgType.CF_ENTITY_LIFECYCLE_MSG; + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitMsg.java new file mode 100644 index 0000000000..e453d2963c --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitMsg.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; + +@Data +public class CalculatedFieldInitMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final CalculatedField cf; + + @Override + public MsgType getMsgType() { + return MsgType.CF_INIT_MSG; + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldLinkInitMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldLinkInitMsg.java new file mode 100644 index 0000000000..d142eb78d8 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldLinkInitMsg.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; + +@Data +public class CalculatedFieldLinkInitMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final CalculatedFieldLink link; + + @Override + public MsgType getMsgType() { + return MsgType.CF_LINK_INIT_MSG; + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldPartitionChangeMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldPartitionChangeMsg.java new file mode 100644 index 0000000000..38a4853219 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldPartitionChangeMsg.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; + +import java.util.Set; + +@Data +public class CalculatedFieldPartitionChangeMsg implements ToCalculatedFieldSystemMsg { + + private final boolean[] partitions; + + @Override + public TenantId getTenantId() { + return TenantId.SYS_TENANT_ID; + } + + @Override + public MsgType getMsgType() { + return MsgType.CF_PARTITIONS_CHANGE_MSG; + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index b429d503d9..90b82a1e1e 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.msg.plugin; +import lombok.Builder; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; @@ -37,6 +38,25 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { private final TenantId tenantId; private final EntityId entityId; private final ComponentLifecycleEvent event; + private final String oldName; + private final String name; + private final EntityId oldProfileId; + private final EntityId profileId; + + public ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event) { + this(tenantId, entityId, event, null, null, null, null); + } + + @Builder + private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId) { + this.tenantId = tenantId; + this.entityId = entityId; + this.event = event; + this.oldName = oldName; + this.name = name; + this.oldProfileId = oldProfileId; + this.profileId = profileId; + } public Optional getRuleChainId() { return entityId.getEntityType() == EntityType.RULE_CHAIN ? Optional.of((RuleChainId) entityId) : Optional.empty(); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java index c8eed097bf..ee8990d931 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java @@ -15,6 +15,10 @@ */ package org.thingsboard.server.common.msg.queue; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.UUID; + public interface TbCallback { TbCallback EMPTY = new TbCallback() { @@ -30,6 +34,10 @@ public interface TbCallback { } }; + default UUID getId(){ + return EntityId.NULL_UUID; + } + void onSuccess(); void onFailure(Throwable t); diff --git a/common/message/src/main/proto/tbmsg.proto b/common/message/src/main/proto/tbmsg.proto index 86bdef30e7..65a967e9e4 100644 --- a/common/message/src/main/proto/tbmsg.proto +++ b/common/message/src/main/proto/tbmsg.proto @@ -70,4 +70,11 @@ message TbMsgProto { int64 correlationIdMSB = 20; int64 correlationIdLSB = 21; int32 partition = 22; + + repeated CalculatedFieldIdProto calculatedFields = 23; +} + +message CalculatedFieldIdProto { + int64 calculatedFieldIdMSB = 1; + int64 calculatedFieldIdLSB = 2; } diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index f911ef3e6e..a570dbea89 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ResourceSubType; import org.thingsboard.server.common.data.ResourceType; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; @@ -58,12 +59,15 @@ import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.rpc.RpcError; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; @@ -88,6 +92,7 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceDeleteMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; import java.util.ArrayList; import java.util.Arrays; @@ -111,14 +116,28 @@ public class ProtoUtils { } public static TransportProtos.ComponentLifecycleMsgProto toProto(ComponentLifecycleMsg msg) { - return TransportProtos.ComponentLifecycleMsgProto.newBuilder() + var builder = TransportProtos.ComponentLifecycleMsgProto.newBuilder() .setTenantIdMSB(msg.getTenantId().getId().getMostSignificantBits()) .setTenantIdLSB(msg.getTenantId().getId().getLeastSignificantBits()) .setEntityType(toProto(msg.getEntityId().getEntityType())) .setEntityIdMSB(msg.getEntityId().getId().getMostSignificantBits()) .setEntityIdLSB(msg.getEntityId().getId().getLeastSignificantBits()) - .setEvent(TransportProtos.ComponentLifecycleEvent.forNumber(msg.getEvent().ordinal())) - .build(); + .setEvent(TransportProtos.ComponentLifecycleEvent.forNumber(msg.getEvent().ordinal())); + if (msg.getProfileId() != null) { + builder.setProfileIdMSB(msg.getProfileId().getId().getMostSignificantBits()); + builder.setProfileIdLSB(msg.getProfileId().getId().getLeastSignificantBits()); + } + if (msg.getOldProfileId() != null) { + builder.setOldProfileIdMSB(msg.getOldProfileId().getId().getMostSignificantBits()); + builder.setOldProfileIdLSB(msg.getOldProfileId().getId().getLeastSignificantBits()); + } + if (msg.getName() != null) { + builder.setName(msg.getName()); + } + if (msg.getOldName() != null) { + builder.setName(msg.getOldName()); + } + return builder.build(); } public static TransportProtos.EntityTypeProto toProto(EntityType entityType) { @@ -126,11 +145,26 @@ public class ProtoUtils { } public static ComponentLifecycleMsg fromProto(TransportProtos.ComponentLifecycleMsgProto proto) { - return new ComponentLifecycleMsg( - TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), - EntityIdFactory.getByTypeAndUuid(fromProto(proto.getEntityType()), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())), - ComponentLifecycleEvent.values()[proto.getEventValue()] - ); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(fromProto(proto.getEntityType()), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + var builder = ComponentLifecycleMsg.builder() + .tenantId(TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB()))) + .entityId(entityId) + .event(ComponentLifecycleEvent.values()[proto.getEventValue()]); + if (!StringUtils.isEmpty(proto.getName())) { + builder.name(proto.getName()); + } + if (!StringUtils.isEmpty(proto.getOldName())) { + builder.oldName(proto.getOldName()); + } + if (proto.getProfileIdMSB() != 0 || proto.getProfileIdLSB() != 0) { + var profileType = EntityType.DEVICE.equals(entityId.getEntityType()) ? EntityType.DEVICE_PROFILE : EntityType.ASSET_PROFILE; + builder.profileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB()))); + } + if (proto.getOldProfileIdMSB() != 0 || proto.getOldProfileIdLSB() != 0) { + var profileType = EntityType.DEVICE.equals(entityId.getEntityType()) ? EntityType.DEVICE_PROFILE : EntityType.ASSET_PROFILE; + builder.oldProfileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB()))); + } + return builder.build(); } public static EntityType fromProto(TransportProtos.EntityTypeProto entityType) { @@ -627,6 +661,96 @@ public class ProtoUtils { return new BaseAttributeKvEntry(entry, proto.getLastUpdateTs(), proto.hasVersion() ? proto.getVersion() : null); } + public static BasicKvEntry basicKvEntryFromProto(TransportProtos.AttributeValueProto proto) { + boolean hasValue = proto.getHasV(); + String key = proto.getKey(); + return switch (proto.getType()) { + case BOOLEAN_V -> new BooleanDataEntry(key, hasValue ? proto.getBoolV() : null); + case LONG_V -> new LongDataEntry(key, hasValue ? proto.getLongV() : null); + case DOUBLE_V -> new DoubleDataEntry(key, hasValue ? proto.getDoubleV() : null); + case STRING_V -> new StringDataEntry(key, hasValue ? proto.getStringV() : null); + case JSON_V -> new JsonDataEntry(key, hasValue ? proto.getJsonV() : null); + default -> null; + }; + } + + public static BasicKvEntry fromProto(KeyValueProto proto) { + String key = proto.getKey(); + return switch (proto.getType()) { + case BOOLEAN_V -> new BooleanDataEntry(key, proto.getBoolV()); + case LONG_V -> new LongDataEntry(key, proto.getLongV()); + case DOUBLE_V -> new DoubleDataEntry(key, proto.getDoubleV()); + case STRING_V -> new StringDataEntry(key, proto.getStringV()); + case JSON_V -> new JsonDataEntry(key, proto.getJsonV()); + default -> null; + }; + } + + public static BasicKvEntry basicKvEntryFromKvEntry(KvEntry kvEntry) { + String key = kvEntry.getKey(); + return switch (kvEntry.getDataType()) { + case BOOLEAN -> new BooleanDataEntry(key, kvEntry.getBooleanValue().orElse(null)); + case LONG -> new LongDataEntry(key, kvEntry.getLongValue().orElse(null)); + case DOUBLE -> new DoubleDataEntry(key, kvEntry.getDoubleValue().orElse(null)); + case STRING -> new StringDataEntry(key, kvEntry.getStrValue().orElse(null)); + case JSON -> new JsonDataEntry(key, kvEntry.getJsonValue().orElse(null)); + }; + } + + public static TsKvEntry fromProto(TransportProtos.TsKvProto proto) { + TransportProtos.KeyValueProto kvProto = proto.getKv(); + String key = kvProto.getKey(); + KvEntry entry = switch (kvProto.getType()) { + case BOOLEAN_V -> new BooleanDataEntry(key, kvProto.getBoolV()); + case LONG_V -> new LongDataEntry(key, kvProto.getLongV()); + case DOUBLE_V -> new DoubleDataEntry(key, kvProto.getDoubleV()); + case STRING_V -> new StringDataEntry(key, kvProto.getStringV()); + case JSON_V -> new JsonDataEntry(key, kvProto.getJsonV()); + default -> null; + }; + return new BasicTsKvEntry(proto.getTs(), entry, proto.hasVersion() ? proto.getVersion() : null); + } + + public static TransportProtos.TsKvProto toTsKvProto(TsKvEntry tsKvEntry) { + var builder = TransportProtos.TsKvProto.newBuilder() + .setTs(tsKvEntry.getTs()) + .setKv(toKeyValueProto(tsKvEntry)); + if (tsKvEntry.getVersion() != null) { + builder.setVersion(tsKvEntry.getVersion()); + } + return builder.build(); + } + + public static TransportProtos.KeyValueProto toKeyValueProto(KvEntry kvEntry) { + TransportProtos.KeyValueProto.Builder builder = TransportProtos.KeyValueProto.newBuilder(); + builder.setKey(kvEntry.getKey()); + switch (kvEntry.getDataType()) { + case BOOLEAN: + builder.setType(TransportProtos.KeyValueType.BOOLEAN_V) + .setBoolV(kvEntry.getBooleanValue().orElse(false)); + break; + case LONG: + builder.setType(TransportProtos.KeyValueType.LONG_V) + .setLongV(kvEntry.getLongValue().orElse(0L)); + break; + case DOUBLE: + builder.setType(TransportProtos.KeyValueType.DOUBLE_V) + .setDoubleV(kvEntry.getDoubleValue().orElse(0.0)); + break; + case STRING: + builder.setType(TransportProtos.KeyValueType.STRING_V) + .setStringV(kvEntry.getStrValue().orElse("")); + break; + case JSON: + builder.setType(TransportProtos.KeyValueType.JSON_V) + .setJsonV(kvEntry.getJsonValue().orElse("{}")); + break; + default: + throw new IllegalArgumentException("Unsupported KvEntry data type: " + kvEntry.getDataType()); + } + return builder.build(); + } + public static TransportProtos.DeviceProto toProto(Device device) { var builder = TransportProtos.DeviceProto.newBuilder() .setTenantIdMSB(device.getTenantId().getId().getMostSignificantBits()) @@ -1019,7 +1143,9 @@ public class ProtoUtils { .setTbelExecState(apiUsageState.getTbelExecState().name()) .setEmailExecState(apiUsageState.getEmailExecState().name()) .setSmsExecState(apiUsageState.getSmsExecState().name()) - .setAlarmExecState(apiUsageState.getAlarmExecState().name()).build(); + .setAlarmExecState(apiUsageState.getAlarmExecState().name()) + .setVersion(apiUsageState.getVersion()) + .build(); } public static ApiUsageState fromProto(TransportProtos.ApiUsageStateProto proto) { @@ -1035,6 +1161,7 @@ public class ProtoUtils { apiUsageState.setEmailExecState(ApiUsageStateValue.valueOf(proto.getEmailExecState())); apiUsageState.setSmsExecState(ApiUsageStateValue.valueOf(proto.getSmsExecState())); apiUsageState.setAlarmExecState(ApiUsageStateValue.valueOf(proto.getAlarmExecState())); + apiUsageState.setVersion(proto.getVersion()); return apiUsageState; } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 25304ad77a..c6708b61c6 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -59,6 +59,8 @@ enum EntityTypeProto { DOMAIN = 36; MOBILE_APP = 37; MOBILE_APP_BUNDLE = 38; + CALCULATED_FIELD = 39; + CALCULATED_FIELD_LINK = 40; } /** @@ -356,6 +358,7 @@ message ApiUsageStateProto { string emailExecState = 14; string smsExecState = 15; string alarmExecState = 16; + int64 version = 17; } message RepositorySettingsProto { @@ -806,6 +809,68 @@ message DeviceInactivityProto { int64 lastInactivityTime = 5; } +message CalculatedFieldTelemetryMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + string entityType = 3; + int64 entityIdMSB = 4; + int64 entityIdLSB = 5; + repeated CalculatedFieldIdProto previousCalculatedFields = 7; + repeated TsKvProto tsData = 9; + AttributeScopeProto scope = 10; + repeated AttributeValueProto attrData = 11; + repeated string removedTsKeys = 12; + repeated string removedAttrKeys = 13; + int64 tbMsgIdMSB = 14; + int64 tbMsgIdLSB = 15; + string tbMsgType = 16; +} + +message CalculatedFieldLinkedTelemetryMsgProto { + CalculatedFieldTelemetryMsgProto msg = 1; + repeated CalculatedFieldEntityCtxIdProto links = 2; +} + +message CalculatedFieldEntityCtxIdProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; + string entityType = 5; + int64 entityIdMSB = 6; + int64 entityIdLSB = 7; +} + +message CalculatedFieldIdProto { + int64 calculatedFieldIdMSB = 1; + int64 calculatedFieldIdLSB = 2; +} + +message SingleValueArgumentProto { + string argName = 1; + TsValueProto value = 2; + int64 version = 3; +} + +message TsDoubleValProto { + int64 ts = 1; + double value = 2; +} + +message TsRollingArgumentProto { + string key = 1; + int32 limit = 2; + int64 timeWindow = 3; + repeated TsDoubleValProto tsValue = 4; +} + +message CalculatedFieldStateProto { + CalculatedFieldEntityCtxIdProto id = 1; + string type = 2; + repeated SingleValueArgumentProto singleValueArguments = 3; + repeated TsRollingArgumentProto rollingValueArguments = 4; +} + //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; @@ -1161,6 +1226,13 @@ message ComponentLifecycleMsgProto { int64 entityIdMSB = 4; int64 entityIdLSB = 5; ComponentLifecycleEvent event = 6; + //Since 4.0. TODO: replace the DeviceNameOrTypeUpdateMsgProto + string oldName = 7; + string name = 8; + int64 oldProfileIdMSB = 9; + int64 oldProfileIdLSB = 10; + int64 profileIdMSB = 11; + int64 profileIdLSB = 12; } message EdgeEventMsgProto { @@ -1588,6 +1660,17 @@ message ToEdgeEventNotificationMsg { EdgeEventMsgProto edgeEventMsg = 1; } +message ToCalculatedFieldMsg { + ComponentLifecycleMsgProto componentLifecycleMsg = 1; + CalculatedFieldTelemetryMsgProto telemetryMsg = 2; + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 3; +} + +message ToCalculatedFieldNotificationMsg { + ComponentLifecycleMsgProto componentLifecycleMsg = 1; + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 2; +} + /* Messages that are handled by ThingsBoard RuleEngine Service */ message ToRuleEngineMsg { int64 tenantIdMSB = 1; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 2fbc3e45b7..7e7de64a5c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -118,9 +118,9 @@ public abstract class AbstractTbQueueConsumerTemplate i if (record != null) { result.add(decode(record)); } - } catch (IOException e) { - log.error("Failed decode record: [{}]", record); - throw new RuntimeException("Failed to decode record: ", e); + } catch (Exception e) { + log.error("Failed to decode record {}", record, e); + throw new RuntimeException("Failed to decode record " + record, e); } }); return result; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java index 71cf4bc1a5..55e07e5856 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java @@ -24,35 +24,36 @@ public class AbstractTbQueueTemplate { protected static final String RESPONSE_TOPIC_HEADER = "responseTopic"; protected static final String EXPIRE_TS_HEADER = "expireTs"; - protected byte[] uuidToBytes(UUID uuid) { + public static byte[] uuidToBytes(UUID uuid) { ByteBuffer buf = ByteBuffer.allocate(16); buf.putLong(uuid.getMostSignificantBits()); buf.putLong(uuid.getLeastSignificantBits()); return buf.array(); } - protected static UUID bytesToUuid(byte[] bytes) { + public static UUID bytesToUuid(byte[] bytes) { ByteBuffer bb = ByteBuffer.wrap(bytes); long firstLong = bb.getLong(); long secondLong = bb.getLong(); return new UUID(firstLong, secondLong); } - protected byte[] stringToBytes(String string) { + public static byte[] stringToBytes(String string) { return string.getBytes(StandardCharsets.UTF_8); } - protected String bytesToString(byte[] data) { + public static String bytesToString(byte[] data) { return new String(data, StandardCharsets.UTF_8); } - protected static byte[] longToBytes(long x) { + public static byte[] longToBytes(long x) { ByteBuffer longBuffer = ByteBuffer.allocate(Long.BYTES); longBuffer.putLong(0, x); return longBuffer.array(); } - protected static long bytesToLong(byte[] bytes) { + public static long bytesToLong(byte[] bytes) { return ByteBuffer.wrap(bytes).getLong(); } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java index 4a5caa35b4..a5bf6c8861 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java @@ -50,6 +50,7 @@ public class TbProtoQueueMsg i @Override public byte[] getData() { - return value.toByteArray(); + return value != null ? value.toByteArray() : null; } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java index 8e2efe757b..14394bbbe9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java @@ -250,7 +250,9 @@ public class MainQueueConsumerManager msgs, TbQueueConsumer consumer, C config) throws Exception { + log.trace("Processing {} messages", msgs.size()); msgPackProcessor.process(msgs, consumer, config); + log.trace("Processed {} messages", msgs.size()); } public void stop() { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java index 8650e0c368..8870ff2a2c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java @@ -24,8 +24,8 @@ import java.util.Collections; import java.util.HashSet; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTopic; @@ -40,7 +40,7 @@ public class QueueStateService { private final Set partitionsInProgress = ConcurrentHashMap.newKeySet(); private boolean initialized; - private final Lock lock = new ReentrantLock(); + private final ReadWriteLock partitionsLock = new ReentrantReadWriteLock(); public void init(PartitionedQueueConsumerManager stateConsumer, PartitionedQueueConsumerManager eventConsumer) { this.stateConsumer = stateConsumer; @@ -49,7 +49,8 @@ public class QueueStateService { public void update(Set newPartitions) { newPartitions = withTopic(newPartitions, stateConsumer.getTopic()); - lock.lock(); + var writeLock = partitionsLock.writeLock(); + writeLock.lock(); Set oldPartitions = this.partitions != null ? this.partitions : Collections.emptySet(); Set addedPartitions; Set removedPartitions; @@ -60,7 +61,7 @@ public class QueueStateService { removedPartitions.removeAll(newPartitions); this.partitions = newPartitions; } finally { - lock.unlock(); + writeLock.unlock(); } if (!removedPartitions.isEmpty()) { @@ -71,7 +72,8 @@ public class QueueStateService { if (!addedPartitions.isEmpty()) { partitionsInProgress.addAll(addedPartitions); stateConsumer.addPartitions(addedPartitions, partition -> { - lock.lock(); + var readLock = partitionsLock.readLock(); + readLock.lock(); try { partitionsInProgress.remove(partition); log.info("Finished partition {} (still in progress: {})", partition, partitionsInProgress); @@ -82,7 +84,7 @@ public class QueueStateService { eventConsumer.addPartitions(Set.of(partition.withTopic(eventConsumer.getTopic()))); } } finally { - lock.unlock(); + readLock.unlock(); } }); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index 0f7affcd40..609d3f8eee 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -40,12 +40,7 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; -import static org.thingsboard.common.util.SystemUtil.getCpuCount; -import static org.thingsboard.common.util.SystemUtil.getCpuUsage; -import static org.thingsboard.common.util.SystemUtil.getDiscSpaceUsage; -import static org.thingsboard.common.util.SystemUtil.getMemoryUsage; -import static org.thingsboard.common.util.SystemUtil.getTotalDiscSpace; -import static org.thingsboard.common.util.SystemUtil.getTotalMemory; +import static org.thingsboard.common.util.SystemUtil.*; @Component diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 09627f3704..b42f8cc380 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -63,6 +63,12 @@ public class HashPartitionService implements PartitionService { private String coreTopic; @Value("${queue.core.partitions:10}") private Integer corePartitions; + @Value("${queue.calculated_fields.event_topic:tb_cf_event}") + private String cfEventTopic; + @Value("${queue.calculated_fields.state_topic:tb_cf_state}") + private String cfStateTopic; + @Value("${queue.calculated_fields.partitions:10}") + private Integer cfPartitions; @Value("${queue.vc.topic:tb_version_control}") private String vcTopic; @Value("${queue.vc.partitions:10}") @@ -111,10 +117,16 @@ public class HashPartitionService implements PartitionService { @PostConstruct public void init() { this.hashFunction = forName(hashFunctionName); + QueueKey coreKey = new QueueKey(ServiceType.TB_CORE); partitionSizesMap.put(coreKey, corePartitions); partitionTopicsMap.put(coreKey, coreTopic); + partitionSizesMap.put(QueueKey.CF, cfPartitions); + partitionTopicsMap.put(QueueKey.CF, cfEventTopic); + partitionSizesMap.put(QueueKey.CF_STATES, cfPartitions); + partitionTopicsMap.put(QueueKey.CF_STATES, cfStateTopic); + QueueKey vcKey = new QueueKey(ServiceType.TB_VC_EXECUTOR); partitionSizesMap.put(vcKey, vcPartitions); partitionTopicsMap.put(vcKey, vcTopic); @@ -144,6 +156,11 @@ public class HashPartitionService implements PartitionService { return myPartitions.get(queueKey); } + @Override + public String getTopic(QueueKey queueKey) { + return partitionTopicsMap.get(queueKey); + } + private void doInitRuleEnginePartitions() { List queueRoutingInfoList = getQueueRoutingInfos(); queueRoutingInfoList.forEach(queue -> { @@ -319,7 +336,8 @@ public class HashPartitionService implements PartitionService { } } - private TopicPartitionInfo resolve(QueueKey queueKey, EntityId entityId) { + @Override + public TopicPartitionInfo resolve(QueueKey queueKey, EntityId entityId) { Integer partitionSize = partitionSizesMap.get(queueKey); if (partitionSize == null) { throw new IllegalStateException("Partitions info for queue " + queueKey + " is missing"); @@ -517,7 +535,6 @@ public class HashPartitionService implements PartitionService { return result; } - @Override public int resolvePartitionIndex(UUID entityId, int partitions) { int hash = hash(entityId); @@ -535,6 +552,11 @@ public class HashPartitionService implements PartitionService { return list == null ? 0 : list.size(); } + @Override + public int getTotalCalculatedFieldPartitions() { + return cfPartitions; + } + private Map> getServiceKeyListMap(List services) { final Map> currentMap = new HashMap<>(); services.forEach(serviceInfo -> { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index 1c680cb478..404b0258c0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -37,6 +37,8 @@ public interface PartitionService { TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); + TopicPartitionInfo resolve(QueueKey queueKey, EntityId entityId); + List resolveAll(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId); boolean isMyPartition(ServiceType serviceType, TenantId tenantId, EntityId entityId); @@ -45,6 +47,8 @@ public interface PartitionService { List getMyPartitions(QueueKey queueKey); + String getTopic(QueueKey queueKey); + /** * Received from the Discovery service when network topology is changed. * @param currentService - current service information {@link org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo} @@ -77,4 +81,6 @@ public interface PartitionService { int resolvePartitionIndex(UUID entityId, int partitions); + int getTotalCalculatedFieldPartitions(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java index 1709003ada..ca38959fdd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java @@ -23,6 +23,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; +import static org.thingsboard.server.common.data.DataConstants.CF_QUEUE_NAME; +import static org.thingsboard.server.common.data.DataConstants.CF_STATES_QUEUE_NAME; + @Data @AllArgsConstructor public class QueueKey { @@ -32,6 +35,9 @@ public class QueueKey { private final String queueName; private final TenantId tenantId; + public static final QueueKey CF = new QueueKey(ServiceType.TB_RULE_ENGINE).withQueueName(CF_QUEUE_NAME); + public static final QueueKey CF_STATES = new QueueKey(ServiceType.TB_RULE_ENGINE).withQueueName(CF_STATES_QUEUE_NAME); + public QueueKey(ServiceType type, Queue queue) { this.type = type; this.queueName = queue.getName(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java index f3b52cf23f..81be9e2207 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java @@ -35,6 +35,7 @@ public class TopicService { private final ConcurrentMap tbCoreNotificationTopics = new ConcurrentHashMap<>(); private final ConcurrentMap tbRuleEngineNotificationTopics = new ConcurrentHashMap<>(); private final ConcurrentMap tbEdgeNotificationTopics = new ConcurrentHashMap<>(); + private final ConcurrentMap tbCalculatedFieldNotificationTopics = new ConcurrentHashMap<>(); private final ConcurrentReferenceHashMap tbEdgeEventsNotificationTopics = new ConcurrentReferenceHashMap<>(); /** @@ -62,6 +63,11 @@ public class TopicService { return buildTopicPartitionInfo("tb_edge.notifications." + serviceId, null, null, false); } + public TopicPartitionInfo getCalculatedFieldNotificationsTopic(String serviceId) { + return tbCalculatedFieldNotificationTopics.computeIfAbsent(serviceId, + id -> buildNotificationsTopicPartitionInfo("calculated_field", serviceId)); + } + public TopicPartitionInfo getEdgeEventNotificationsTopic(TenantId tenantId, EdgeId edgeId) { return tbEdgeEventsNotificationTopics.computeIfAbsent(edgeId, id -> buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId)); } @@ -71,7 +77,11 @@ public class TopicService { } private TopicPartitionInfo buildNotificationsTopicPartitionInfo(ServiceType serviceType, String serviceId) { - return buildTopicPartitionInfo(serviceType.name().toLowerCase() + ".notifications." + serviceId, null, null, false); + return buildNotificationsTopicPartitionInfo(serviceType.name().toLowerCase(), serviceId); + } + + private TopicPartitionInfo buildNotificationsTopicPartitionInfo(String serviceType, String serviceId) { + return buildTopicPartitionInfo(serviceType + ".notifications." + serviceId, null, null, false); } public TopicPartitionInfo buildTopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean myPartition) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index 3ad6164c61..f7a4d2abf6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -251,25 +251,21 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi } } catch (Exception e) { log.error("Failed to delete ZK node {}", nodePath, e); - throw new RuntimeException(e); } } private void destroyZkClient() { stopped = true; - try { - unpublishCurrentServer(); - } catch (Exception e) { - } + unpublishCurrentServer(); CloseableUtils.closeQuietly(cache); CloseableUtils.closeQuietly(client); log.info("ZK client disconnected"); } @PreDestroy - public void destroy() { - destroyZkClient(); + private void destroy() { zkExecutorService.shutdownNow(); + destroyZkClient(); log.info("Stopped discovery service"); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java index e7b126a6a0..597463300a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java @@ -24,6 +24,7 @@ import org.thingsboard.server.queue.discovery.QueueKey; import java.io.Serial; import java.util.Collection; +import java.util.Collections; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -62,6 +63,10 @@ public class PartitionChangeEvent extends TbApplicationEvent { return newPartitions.values().stream().flatMap(Collection::stream).collect(Collectors.toSet()); } + public Set getCfPartitions() { + return newPartitions.getOrDefault(QueueKey.CF, Collections.emptySet()); + } + private Set getPartitionsByServiceTypeAndQueueName(ServiceType serviceType, String queueName) { return newPartitions.entrySet() .stream() diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java index cec3e3752c..ffd4321060 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java @@ -23,18 +23,19 @@ import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders; import java.util.UUID; public class KafkaTbQueueMsg implements TbQueueMsg { + + private static final int UUID_LENGTH = 36; + private final UUID key; private final TbQueueMsgHeaders headers; private final byte[] data; public KafkaTbQueueMsg(ConsumerRecord record) { - UUID key; - try { - key = UUID.fromString(record.key()); - } catch (IllegalArgumentException e) { - key = null; // FIXME + if (record.key().length() <= UUID_LENGTH) { + this.key = UUID.fromString(record.key()); + } else { + this.key = UUID.randomUUID(); } - this.key = key; TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); record.headers().forEach(header -> { headers.put(header.key(), header.value()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index bb385eb43e..d219428941 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.kafka; import lombok.Builder; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; @@ -31,6 +32,7 @@ import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; import java.io.IOException; import java.time.Duration; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -57,8 +59,6 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue private int readCount; private Map endOffsets; // needed if stopWhenRead is true - private boolean partitionsAssigned = false; - @Builder private TbKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, String clientId, String groupId, String topic, @@ -107,18 +107,28 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue if (kafkaPartitions == null) { toSubscribe.add(topic); } else { - consumer.assign(kafkaPartitions.stream() + List topicPartitions = kafkaPartitions.stream() .map(partition -> new TopicPartition(topic, partition)) - .toList()); - partitionsAssigned = true; - onPartitionsAssigned(); + .toList(); + consumer.assign(topicPartitions); + onPartitionsAssigned(topicPartitions); } }); if (!toSubscribe.isEmpty()) { - consumer.subscribe(toSubscribe); - } - if (readFromBeginning) { - consumer.seekToBeginning(Collections.emptySet()); // for all assigned partitions + if (readFromBeginning || stopWhenRead) { + consumer.subscribe(toSubscribe, new ConsumerRebalanceListener() { + @Override + public void onPartitionsRevoked(Collection partitions) {} + + @Override + public void onPartitionsAssigned(Collection partitions) { + log.debug("Handling onPartitionsAssigned {}", partitions); + TbKafkaConsumerTemplate.this.onPartitionsAssigned(partitions); + } + }); + } else { + consumer.subscribe(toSubscribe); + } } } else { log.info("unsubscribe due to empty topic list"); @@ -134,13 +144,6 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue log.trace("poll topic {} maxDuration {}", getTopic(), durationInMillis); ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); - if (!partitionsAssigned) { - if (readFromBeginning) { - consumer.seekToBeginning(Collections.emptySet()); - } - partitionsAssigned = true; - onPartitionsAssigned(); - } stopWatch.stop(); log.trace("poll topic {} took {}ms", getTopic(), stopWatch.getTotalTimeMillis()); @@ -152,7 +155,7 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue recordList = new ArrayList<>(256); records.forEach(record -> { recordList.add(record); - if (stopWhenRead) { + if (stopWhenRead && endOffsets != null) { readCount++; int partition = record.partition(); Long endOffset = endOffsets.get(partition); @@ -167,16 +170,19 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue } }); } - if (stopWhenRead && endOffsets.isEmpty()) { + if (stopWhenRead && endOffsets != null && endOffsets.isEmpty()) { log.info("Finished reading {}, processed {} messages", partitions, readCount); stop(); } return recordList; } - private void onPartitionsAssigned() { + private void onPartitionsAssigned(Collection partitions) { + if (readFromBeginning) { + consumer.seekToBeginning(partitions); + } if (stopWhenRead) { - endOffsets = consumer.endOffsets(consumer.assignment()).entrySet().stream() + endOffsets = consumer.endOffsets(partitions).entrySet().stream() .filter(entry -> entry.getValue() > 0) .collect(Collectors.toMap(entry -> entry.getKey().partition(), Map.Entry::getValue)); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java index 00926394b3..cac6f2ea1e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java @@ -121,7 +121,7 @@ public class TbKafkaProducerTemplate implements TbQueuePro if (callback != null) { callback.onFailure(exception); } else { - log.warn("Producer template failure: {}", exception.getMessage(), exception); + log.warn("Producer template failure", exception); } } }); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index 02c30c7848..aebda5a5bc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -52,6 +52,10 @@ public class TbKafkaTopicConfigs { private String housekeeperProperties; @Value("${queue.kafka.topic-properties.housekeeper-reprocessing:}") private String housekeeperReprocessingProperties; + @Value("${queue.kafka.topic-properties.calculated-field:}") + private String calculatedFieldProperties; + @Value("${queue.kafka.topic-properties.calculated-field-state:}") + private String calculatedFieldStateProperties; @Value("${queue.kafka.topic-properties.edqs-events:}") private String edqsEventsProperties; @Value("${queue.kafka.topic-properties.edqs-requests:}") @@ -86,6 +90,10 @@ public class TbKafkaTopicConfigs { @Getter private Map edgeEventConfigs; @Getter + private Map calculatedFieldConfigs; + @Getter + private Map calculatedFieldStateConfigs; + @Getter private Map edqsEventsConfigs; @Getter private Map edqsRequestsConfigs; @@ -109,6 +117,8 @@ public class TbKafkaTopicConfigs { housekeeperReprocessingConfigs = PropertyUtils.getProps(housekeeperReprocessingProperties); edgeConfigs = PropertyUtils.getProps(edgeProperties); edgeEventConfigs = PropertyUtils.getProps(edgeEventProperties); + calculatedFieldConfigs = PropertyUtils.getProps(calculatedFieldProperties); + calculatedFieldStateConfigs = PropertyUtils.getProps(calculatedFieldStateProperties); edqsEventsConfigs = PropertyUtils.getProps(edqsEventsProperties); edqsRequestsConfigs = PropertyUtils.getProps(edqsRequestsProperties); edqsStateConfigs = PropertyUtils.getProps(edqsStateProperties); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index df5dc15141..e97af10ecc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -40,6 +41,7 @@ import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.memory.InMemoryStorage; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; @@ -62,6 +64,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; + private final TbQueueCalculatedFieldSettings calculatedFieldSettings; private final EdqsConfig edqsConfig; private final InMemoryStorage storage; @@ -130,6 +133,31 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return null; } + @Override + public TbQueueConsumer> createToCalculatedFieldMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + } + + @Override + public TbQueueProducer> createToCalculatedFieldMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + } + + @Override + public TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.getCalculatedFieldNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); + } + + @Override + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); + } + + @Override + public TbQueueProducer> createCalculatedFieldStateProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); + } + @Override public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); @@ -200,6 +228,11 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return null; } + @Override + public TbQueueProducer> createToCalculatedFieldNotificationMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + } + @Override public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { return new InMemoryTbQueueProducer<>(storage, queue.getTopic()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 86f8c0f1df..bb9c1a028e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -25,7 +25,10 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -58,6 +61,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; @@ -83,6 +87,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueVersionControlSettings vcSettings; private final TbQueueEdgeSettings edgeSettings; + private final TbQueueCalculatedFieldSettings calculatedFieldSettings; private final TbKafkaConsumerStatsService consumerStatsService; private final EdqsConfig edqsConfig; @@ -99,6 +104,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin housekeeperReprocessingAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin cfAdmin; + private final TbQueueAdmin cfStateAdmin; private final TbQueueAdmin edqsEventsAdmin; private final TbKafkaAdmin edqsRequestsAdmin; @@ -114,6 +121,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbQueueVersionControlSettings vcSettings, TbQueueEdgeSettings edgeSettings, + TbQueueCalculatedFieldSettings calculatedFieldSettings, TbKafkaConsumerStatsService consumerStatsService, TbKafkaTopicConfigs kafkaTopicConfigs, EdqsConfig edqsConfig) { @@ -128,6 +136,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.vcSettings = vcSettings; this.consumerStatsService = consumerStatsService; this.edgeSettings = edgeSettings; + this.calculatedFieldSettings = calculatedFieldSettings; this.edqsConfig = edqsConfig; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); @@ -143,6 +152,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); + this.cfStateAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldStateConfigs()); this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsEventsConfigs()); this.edqsRequestsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsRequestsConfigs()); } @@ -502,6 +513,77 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return requestBuilder.build(); } + @Override + public TbQueueConsumer> createToCalculatedFieldMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + consumerBuilder.clientId("monolith-calculated-field-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.groupId(topicService.buildTopicName("monolith-calculated-field-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(cfAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToCalculatedFieldMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-calculated-field-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(cfAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.getCalculatedFieldNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("monolith-calculated-field-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId(topicService.buildTopicName("monolith-calculated-field-notifications-consumer-" + serviceInfoProvider.getServiceId())); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToCalculatedFieldNotificationMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-calculated-field-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(notificationAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())) + .readFromBeginning(true) + .stopWhenRead(true) + .clientId("monolith-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()) + .groupId(topicService.buildTopicName("monolith-calculated-field-state-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), msg.getData() != null ? CalculatedFieldStateProto.parseFrom(msg.getData()) : null, msg.getHeaders())) + .admin(cfStateAdmin) + .statsService(consumerStatsService) + .build(); + } + + @Override + public TbQueueProducer> createCalculatedFieldStateProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("monolith-calculated-field-state-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())) + .admin(cfStateAdmin) + .build(); + } + @Override public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { return TbKafkaProducerTemplate.>builder() @@ -570,6 +652,9 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi if (edgeAdmin != null) { edgeAdmin.destroy(); } + if (cfAdmin != null) { + cfAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index eda3b50645..f3adb266a4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -25,6 +25,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -57,6 +59,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; @@ -83,6 +86,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; + private final TbQueueCalculatedFieldSettings calculatedFieldSettings; private final EdqsConfig edqsConfig; private final TbQueueAdmin coreAdmin; @@ -98,6 +102,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin housekeeperReprocessingAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin cfAdmin; private final TbQueueAdmin edqsEventsAdmin; private final TbKafkaAdmin edqsRequestsAdmin; @@ -115,6 +120,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueEdgeSettings edgeSettings, TbKafkaConsumerStatsService consumerStatsService, TbQueueTransportNotificationSettings transportNotificationSettings, + TbQueueCalculatedFieldSettings calculatedFieldSettings, EdqsConfig edqsConfig, TbKafkaTopicConfigs kafkaTopicConfigs) { this.topicService = topicService; @@ -128,6 +134,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.consumerStatsService = consumerStatsService; this.transportNotificationSettings = transportNotificationSettings; this.edgeSettings = edgeSettings; + this.calculatedFieldSettings = calculatedFieldSettings; this.edqsConfig = edqsConfig; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); @@ -143,6 +150,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsEventsConfigs()); this.edqsRequestsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsRequestsConfigs()); } @@ -451,6 +459,26 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createToCalculatedFieldMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-to-calculated-field-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(cfAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createToCalculatedFieldNotificationMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-calculated-field-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(notificationAdmin); + return requestBuilder.build(); + } + @Override public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { return TbKafkaProducerTemplate.>builder() @@ -516,6 +544,9 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { if (vcAdmin != null) { vcAdmin.destroy(); } + if (cfAdmin != null) { + cfAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index e28a64304e..43fbb5efeb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -23,7 +23,10 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -52,6 +55,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; @@ -74,6 +78,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; + private final TbQueueCalculatedFieldSettings calculatedFieldSettings; private final TbQueueAdmin coreAdmin; private final TbKafkaAdmin ruleEngineAdmin; @@ -84,6 +89,8 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbQueueAdmin housekeeperAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin cfAdmin; + private final TbQueueAdmin cfStateAdmin; private final TbQueueAdmin edqsEventsAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -94,7 +101,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbKafkaConsumerStatsService consumerStatsService, TbQueueTransportNotificationSettings transportNotificationSettings, - TbQueueEdgeSettings edgeSettings, + TbQueueEdgeSettings edgeSettings, TbQueueCalculatedFieldSettings calculatedFieldSettings, TbKafkaTopicConfigs kafkaTopicConfigs) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; @@ -105,6 +112,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.consumerStatsService = consumerStatsService; this.transportNotificationSettings = transportNotificationSettings; this.edgeSettings = edgeSettings; + this.calculatedFieldSettings = calculatedFieldSettings; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -115,6 +123,8 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); + this.cfStateAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldStateConfigs()); this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsEventsConfigs()); } @@ -298,6 +308,67 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { .build(); } + @Override + public TbQueueConsumer> createToCalculatedFieldMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + consumerBuilder.clientId("tb-rule-engine-calculated-field-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.groupId(topicService.buildTopicName("tb-rule-engine-calculated-field-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(cfAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToCalculatedFieldMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-to-calculated-field-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(cfAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.getCalculatedFieldNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("tb-calculated-field-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId(topicService.buildTopicName("tb-calculated-field-notifications-node-") + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())) + .readFromBeginning(true) + .stopWhenRead(true) + .clientId("tb-rule-engine-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()) + .groupId(topicService.buildTopicName("tb-rule-engine-calculated-field-state-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), msg.getData() != null ? CalculatedFieldStateProto.parseFrom(msg.getData()) : null, msg.getHeaders())) + .admin(cfStateAdmin) + .statsService(consumerStatsService) + .build(); + } + + @Override + public TbQueueProducer> createCalculatedFieldStateProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("tb-rule-engine-to-calculated-field-state-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())) + .admin(cfStateAdmin) + .build(); + } + @Override public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { return TbKafkaProducerTemplate.>builder() @@ -332,5 +403,8 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { if (fwUpdatesAdmin != null) { fwUpdatesAdmin.destroy(); } + if (cfAdmin != null) { + cfAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index e2a2b863bf..037d1f2087 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -18,6 +18,8 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -159,4 +161,8 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, Hous return null; } + TbQueueProducer> createToCalculatedFieldMsgProducer(); + + TbQueueProducer> createToCalculatedFieldNotificationMsgProducer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index c65d12dfe6..7c3e415e9f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -17,6 +17,9 @@ package org.thingsboard.server.queue.provider; import jakarta.annotation.PostConstruct; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -48,6 +51,8 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toUsageStats; private TbQueueProducer> toVersionControl; private TbQueueProducer> toHousekeeper; + private TbQueueProducer> toCalculatedFields; + private TbQueueProducer> toCalculatedFieldNotifications; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -66,6 +71,8 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { this.toEdge = tbQueueProvider.createEdgeMsgProducer(); this.toEdgeNotifications = tbQueueProvider.createEdgeNotificationsMsgProducer(); this.toEdgeEvents = tbQueueProvider.createEdgeEventMsgProducer(); + this.toCalculatedFields = tbQueueProvider.createToCalculatedFieldMsgProducer(); + this.toCalculatedFieldNotifications = tbQueueProvider.createToCalculatedFieldNotificationMsgProducer(); } @Override @@ -124,4 +131,14 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { return toEdgeEvents; } + @Override + public TbQueueProducer> getCalculatedFieldsMsgProducer() { + return toCalculatedFields; + } + + @Override + public TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer() { + return toCalculatedFieldNotifications; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index 8b7f19ee5d..865637b2ff 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.queue.provider; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -91,4 +93,8 @@ public interface TbQueueProducerProvider { TbQueueProducer> getTbEdgeEventsMsgProducer(); + TbQueueProducer> getCalculatedFieldsMsgProducer(); + + TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index 8d1ec7f7f6..dcadf02d02 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -18,6 +18,8 @@ package org.thingsboard.server.queue.provider; import jakarta.annotation.PostConstruct; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -47,6 +49,7 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toEdge; private TbQueueProducer> toEdgeNotifications; private TbQueueProducer> toEdgeEvents; + private TbQueueProducer> toCalculatedFields; public TbRuleEngineProducerProvider(TbRuleEngineQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -64,6 +67,7 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { this.toEdge = tbQueueProvider.createEdgeMsgProducer(); this.toEdgeNotifications = tbQueueProvider.createEdgeNotificationsMsgProducer(); this.toEdgeEvents = tbQueueProvider.createEdgeEventMsgProducer(); + this.toCalculatedFields = tbQueueProvider.createToCalculatedFieldMsgProducer(); } @Override @@ -121,4 +125,14 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { return toHousekeeper; } + @Override + public TbQueueProducer> getCalculatedFieldsMsgProducer() { + return toCalculatedFields; + } + + @Override + public TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Rule Engine Service!"); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index a11e047147..767fea9f0c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -17,6 +17,9 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -109,11 +112,22 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory } /** - * Used to consume high priority messages by TB Core Service + * Used to consume high priority messages by TB Rule Engine Service * * @return */ TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer(); TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); + + TbQueueConsumer> createToCalculatedFieldMsgConsumer(); + + TbQueueProducer> createToCalculatedFieldMsgProducer(); + + TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer(); + + TbQueueConsumer> createCalculatedFieldStateConsumer(); + + TbQueueProducer> createCalculatedFieldStateProducer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index e201ddc357..a7a34992cd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import jakarta.annotation.PostConstruct; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -111,4 +112,13 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider return toHousekeeper; } + @Override + public TbQueueProducer> getCalculatedFieldsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } + + @Override + public TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java index d5201e6518..85c400d094 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import jakarta.annotation.PostConstruct; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -107,4 +108,14 @@ public class TbVersionControlProducerProvider implements TbQueueProducerProvider return toHousekeeper; } + @Override + public TbQueueProducer> getCalculatedFieldsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCalculatedFieldSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCalculatedFieldSettings.java new file mode 100644 index 0000000000..c2de8eff4e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCalculatedFieldSettings.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +@Lazy +@Data +@Component +public class TbQueueCalculatedFieldSettings { + + @Value("${queue.calculated_fields.event_topic}") + private String eventTopic; + + @Value("${queue.calculated_fields.state_topic}") + private String stateTopic; + + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java index 8bd58c4e81..46c29b867b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java @@ -38,6 +38,9 @@ public @interface AfterStartUp { int ACTOR_SYSTEM = 9; int REGULAR_SERVICE = 10; + int CF_READ_PROFILE_ENTITIES_SERVICE = 10; + int CF_READ_CF_SERVICE = 11; + int BEFORE_TRANSPORT_SERVICE = Integer.MAX_VALUE - 1001; int TRANSPORT_SERVICE = Integer.MAX_VALUE - 1000; int AFTER_TRANSPORT_SERVICE = Integer.MAX_VALUE - 999; diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index 0f35bad352..9e493220ca 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -37,23 +37,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.*; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.BDDMockito.willAnswer; -import static org.mockito.BDDMockito.willDoNothing; -import static org.mockito.BDDMockito.willReturn; -import static org.mockito.Mockito.RETURNS_DEEP_STUBS; -import static org.mockito.Mockito.atLeastOnce; -import static org.mockito.Mockito.lenient; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; +import static org.mockito.BDDMockito.*; +import static org.mockito.Mockito.*; import static org.mockito.hamcrest.MockitoHamcrest.longThat; @Slf4j diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/discovery/ZkDiscoveryServiceTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/discovery/ZkDiscoveryServiceTest.java index c8d91b0c0f..19643f4ad4 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/discovery/ZkDiscoveryServiceTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/discovery/ZkDiscoveryServiceTest.java @@ -42,11 +42,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; @ExtendWith(MockitoExtension.class) public class ZkDiscoveryServiceTest { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/AbstractScriptInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/AbstractScriptInvokeService.java index efe6fd7781..7cbac0401d 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/AbstractScriptInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/AbstractScriptInvokeService.java @@ -22,6 +22,8 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.script.api.tbel.TbelCfObject; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -256,6 +258,8 @@ public abstract class AbstractScriptInvokeService implements ScriptInvokeService for (Object arg : args) { if (arg instanceof CharSequence) { totalArgsSize += ((CharSequence) arg).length(); + } else if (arg instanceof TbelCfObject tbelCfObj) { + totalArgsSize += tbelCfObj.memorySize(); } else { var str = JacksonUtil.toString(arg); if (str != null) { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java index 1eeed4a5a9..c1d8ae929a 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java @@ -16,5 +16,5 @@ package org.thingsboard.script.api; public enum ScriptType { - RULE_NODE_SCRIPT + RULE_NODE_SCRIPT, CALCULATED_FIELD_SCRIPT } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java index c49e81052c..f626746a64 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java @@ -130,9 +130,13 @@ public class DefaultTbelInvokeService extends AbstractScriptInvokeService implem OptimizerFactory.setDefaultOptimizer(OptimizerFactory.SAFE_REFLECTIVE); parserConfig = ParserContext.enableSandboxedMode(); parserConfig.addImport("JSON", TbJson.class); - parserConfig.registerDataType("Date", TbDate.class, date -> 8L); - parserConfig.registerDataType("Random", Random.class, date -> 8L); - parserConfig.registerDataType("Calendar", Calendar.class, date -> 8L); + parserConfig.registerDataType("Date", TbDate.class, val -> 8L); + parserConfig.registerDataType("Random", Random.class, val -> 8L); + parserConfig.registerDataType("Calendar", Calendar.class, val -> 8L); + parserConfig.registerDataType("TbelCfSingleValueArg", TbelCfSingleValueArg.class, TbelCfSingleValueArg::memorySize); + parserConfig.registerDataType("TbelCfTsRollingArg", TbelCfTsRollingArg.class, TbelCfTsRollingArg::memorySize); + parserConfig.registerDataType("TbelCfTsDoubleVal", TbelCfTsDoubleVal.class, TbelCfTsDoubleVal::memorySize); + parserConfig.registerDataType("TbTimeWindow", TbTimeWindow.class, TbTimeWindow::memorySize); TbUtils.register(parserConfig); executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, "tbel-executor")); try { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbDate.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbDate.java index 4528819e44..c674407a00 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbDate.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbDate.java @@ -37,6 +37,7 @@ import java.time.temporal.ChronoUnit; import java.time.temporal.TemporalAccessor; import java.util.Locale; import java.util.function.BiFunction; +import java.util.function.Function; public class TbDate implements Serializable, Cloneable { @@ -481,6 +482,43 @@ public class TbDate implements Serializable, Cloneable { instant = Instant.ofEpochMilli(dateMilliSecond); } + public void addDays(int days) { + adjustTime(zonedDateTime -> zonedDateTime.plusDays(days)); + } + + public void addYears(int years) { + adjustTime(zonedDateTime -> zonedDateTime.plusYears(years)); + } + + public void addMonths(int months) { + adjustTime(zonedDateTime -> zonedDateTime.plusMonths(months)); + } + + public void addWeeks(int weeks) { + adjustTime(zonedDateTime -> zonedDateTime.plusWeeks(weeks)); + } + + public void addHours(int hours) { + adjustTime(zonedDateTime -> zonedDateTime.plusHours(hours)); + } + + public void addMinutes(int minutes) { + adjustTime(zonedDateTime -> zonedDateTime.plusMinutes(minutes)); + } + + public void addSeconds(int seconds) { + adjustTime(zonedDateTime -> zonedDateTime.plusSeconds(seconds)); + } + + public void addNanos(long nanos) { + adjustTime(zonedDateTime -> zonedDateTime.plusNanos(nanos)); + } + + private void adjustTime(Function adjuster) { + ZonedDateTime zonedDateTime = adjuster.apply(getZonedDateTime()); + this.instant = zonedDateTime.toInstant(); + } + public ZoneOffset getLocaleZoneOffset(Instant... instants){ return ZoneId.systemDefault().getRules().getOffset(instants.length > 0 ? instants[0] : this.instant); } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java new file mode 100644 index 0000000000..ce61965317 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class TbTimeWindow implements TbelCfObject { + + public static final long OBJ_SIZE = 32L; + + private long startTs; + private long endTs; + private int limit; + + @Override + public long memorySize() { + return OBJ_SIZE; + } + +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 9a57240d23..7a54224ddc 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -1299,7 +1299,7 @@ public class TbUtils { if (str == null || str.isEmpty()) { return -1; } - return str.matches("[+-]?\\d+(\\.\\d+)?") ? DEC_RADIX : -1; + return str.matches("[+-]?\\d+(\\.\\d+)?([eE][+-]?\\d+)?") ? DEC_RADIX : -1; } public static int isHexadecimal(String str) { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java new file mode 100644 index 0000000000..f95b08195e --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = TbelCfSingleValueArg.class, name = "SINGLE_VALUE"), + @JsonSubTypes.Type(value = TbelCfTsRollingArg.class, name = "TS_ROLLING") +}) +public interface TbelCfArg extends TbelCfObject { + + @JsonIgnore + String getType(); + +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfObject.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfObject.java new file mode 100644 index 0000000000..3af6198a22 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfObject.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +public interface TbelCfObject { + + long memorySize(); + +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfSingleValueArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfSingleValueArg.java new file mode 100644 index 0000000000..193b6ea1ae --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfSingleValueArg.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class TbelCfSingleValueArg implements TbelCfArg { + + public static final long OBJ_SIZE = 32L; // Approximate calculation; + + private final long ts; + private final Object value; + + @JsonCreator + public TbelCfSingleValueArg( + @JsonProperty("ts") long ts, + @JsonProperty("value") Object value + ) { + this.ts = ts; + this.value = value; + } + + @Override + public long memorySize() { + if (value instanceof String strValue) { + return OBJ_SIZE + strValue.length(); + } else { + return OBJ_SIZE; + } + } + + @Override + public String getType() { + return "SINGLE_VALUE"; + } + +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsDoubleVal.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsDoubleVal.java new file mode 100644 index 0000000000..71565f3e1d --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsDoubleVal.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import lombok.Data; + +@Data +public class TbelCfTsDoubleVal implements TbelCfObject { + + public static final long OBJ_SIZE = 32L; // Approximate calculation; + + private final long ts; + private final double value; + + @Override + public long memorySize() { + return OBJ_SIZE; + } +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java new file mode 100644 index 0000000000..807d498a16 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java @@ -0,0 +1,279 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.function.Consumer; + +import static org.thingsboard.script.api.tbel.TbelCfTsDoubleVal.OBJ_SIZE; + +public class TbelCfTsRollingArg implements TbelCfArg, Iterable { + + @Getter + private final TbTimeWindow timeWindow; + @Getter + private final List values; + + @JsonCreator + public TbelCfTsRollingArg( + @JsonProperty("timeWindow") TbTimeWindow timeWindow, + @JsonProperty("values") List values + ) { + this.timeWindow = timeWindow; + this.values = Collections.unmodifiableList(values); + } + + public TbelCfTsRollingArg(int limit, long timeWindow, List values) { + long ts = System.currentTimeMillis(); + this.timeWindow = new TbTimeWindow(ts - timeWindow, ts, limit); + this.values = Collections.unmodifiableList(values); + } + + @Override + public long memorySize() { + return 12 + values.size() * OBJ_SIZE; + } + + @JsonIgnore + public List getValue() { + return values; + } + + public double max() { + return max(true); + } + + public double max(boolean ignoreNaN) { + if (values.isEmpty()) { + throw new IllegalArgumentException("Rolling argument values are empty."); + } + + double max = Double.MIN_VALUE; + for (TbelCfTsDoubleVal value : values) { + double val = value.getValue(); + if (!ignoreNaN && Double.isNaN(val)) { + return val; + } + if (max < val) { + max = val; + } + } + return max; + } + + public double min() { + return min(true); + } + + public double min(boolean ignoreNaN) { + if (values.isEmpty()) { + throw new IllegalArgumentException("Rolling argument values are empty."); + } + + double min = Double.MAX_VALUE; + for (TbelCfTsDoubleVal value : values) { + double val = value.getValue(); + if (!ignoreNaN && Double.isNaN(val)) { + return Double.NaN; + } + if (min > val) { + min = val; + } + } + return min; + } + + public double mean() { + return mean(true); + } + + public double mean(boolean ignoreNaN) { + if (values.isEmpty()) { + throw new IllegalArgumentException("Rolling argument values are empty."); + } + + return sum(ignoreNaN) / count(ignoreNaN); + } + + public double std() { + return std(true); + } + + public double std(boolean ignoreNaN) { + if (values.isEmpty()) { + throw new IllegalArgumentException("Rolling argument values are empty."); + } + + double mean = mean(ignoreNaN); + if (!ignoreNaN && Double.isNaN(mean)) { + return Double.NaN; + } + + double sum = 0; + for (TbelCfTsDoubleVal value : values) { + double val = value.getValue(); + if (Double.isNaN(val)) { + if (!ignoreNaN) { + return Double.NaN; + } + } else { + sum += Math.pow(val - mean, 2); + } + } + return Math.sqrt(sum / count(ignoreNaN)); + } + + public double median() { + return median(true); + } + + public double median(boolean ignoreNaN) { + if (values.isEmpty()) { + throw new IllegalArgumentException("Rolling argument values are empty."); + } + + List sortedValues = new ArrayList<>(); + for (TbelCfTsDoubleVal value : values) { + double val = value.getValue(); + if (Double.isNaN(val)) { + if (!ignoreNaN) { + return Double.NaN; + } + } else { + sortedValues.add(val); + } + } + Collections.sort(sortedValues); + + int size = sortedValues.size(); + return (size % 2 == 1) + ? sortedValues.get(size / 2) + : (sortedValues.get(size / 2 - 1) + sortedValues.get(size / 2)) / 2.0; + } + + public int count() { + return count(true); + } + + public int count(boolean ignoreNaN) { + int count = 0; + if (ignoreNaN) { + for (TbelCfTsDoubleVal value : values) { + if (!Double.isNaN(value.getValue())) { + count++; + } + } + return count; + } + return values.size(); + } + + public double last() { + return last(true); + } + + public double last(boolean ignoreNaN) { + if (values.isEmpty()) { + throw new IllegalArgumentException("Rolling argument values are empty."); + } + + double value = values.get(values.size() - 1).getValue(); + if (!Double.isNaN(value) || !ignoreNaN) { + return value; + } + for (int i = values.size() - 2; i >= 0; i--) { + double prevValue = values.get(i).getValue(); + if (!Double.isNaN(prevValue)) { + return prevValue; + } + } + throw new IllegalArgumentException("Rolling argument values are empty."); + } + + public double first() { + return first(true); + } + + public double first(boolean ignoreNaN) { + if (values.isEmpty()) { + throw new IllegalArgumentException("Rolling argument values are empty."); + } + + double firstValue = values.get(0).getValue(); + if (!Double.isNaN(firstValue) || !ignoreNaN) { + return firstValue; + } + for (int i = 1; i < values.size(); i++) { + double nextValue = values.get(i).getValue(); + if (!Double.isNaN(nextValue)) { + return nextValue; + } + } + throw new IllegalArgumentException("Rolling argument values are empty."); + } + + public double sum() { + return sum(true); + } + + public double sum(boolean ignoreNaN) { + if (values.isEmpty()) { + throw new IllegalArgumentException("Rolling argument values are empty."); + } + + double sum = 0; + for (TbelCfTsDoubleVal value : values) { + double val = value.getValue(); + if (Double.isNaN(val)) { + if (!ignoreNaN) { + return Double.NaN; + } + } else { + sum += val; + } + } + return sum; + } + + @JsonIgnore + public int getSize() { + return values.size(); + } + + @Override + public Iterator iterator() { + return values.iterator(); + } + + @Override + public void forEach(Consumer action) { + values.forEach(action); + } + + @Override + public String getType() { + return "TS_ROLLING"; + } + +} diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbDateTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbDateTest.java index 9492c2e610..e001e9fb11 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbDateTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbDateTest.java @@ -885,4 +885,64 @@ class TbDateTest { Assertions.assertNotNull(serializedTbDate); Assertions.assertEquals(expectedDate.toString(), serializedTbDate); } + + @Test + void testAddFunctions() { + TbDate d = new TbDate(2024, 1, 1, 10, 0, 0, 0); + testResultChangeDateTime(d); + + d.addYears(1); + testResultChangeDateTime(d); + + d.addYears(-2); + testResultChangeDateTime(d); + + d.addMonths(2); + testResultChangeDateTime(d); + + d.addMonths(10); + testResultChangeDateTime(d); + + d.addMonths(-13); + testResultChangeDateTime(d); + + d.addWeeks(4); + testResultChangeDateTime(d); + + d.addWeeks(-5); + testResultChangeDateTime(d); + + d.addDays(6); + testResultChangeDateTime(d); + + d.addDays(45); + testResultChangeDateTime(d); + + d.addDays(-50); + testResultChangeDateTime(d); + + d.addHours(23); + testResultChangeDateTime(d); + + d.addHours(-47); + testResultChangeDateTime(d); + + d.addMinutes(59); + testResultChangeDateTime(d); + + d.addMinutes(-60); + testResultChangeDateTime(d); + + d.addSeconds(59); + testResultChangeDateTime(d); + + d.addSeconds(-60); + testResultChangeDateTime(d); + + d.addNanos(999999); + testResultChangeDateTime(d); + + d.addNanos(-1000000); + testResultChangeDateTime(d); + } } diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index 72d70e1d92..fbd81948d6 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -442,8 +442,9 @@ public class TbUtilsTest { @Test public void parsDouble() { - String doubleValStr = "1729.1729"; - Assertions.assertEquals(java.util.Optional.of(doubleVal).get(), TbUtils.parseDouble(doubleValStr)); + String doubleValStr = "1.1428250947E8"; + Assertions.assertEquals(Double.parseDouble(doubleValStr), TbUtils.parseDouble(doubleValStr)); + doubleValStr = "1729.1729"; Assertions.assertEquals(0, Double.compare(doubleVal, TbUtils.parseHexToDouble(longValHex))); Assertions.assertEquals(0, Double.compare(doubleValRev, TbUtils.parseHexToDouble(longValHex, false))); Assertions.assertEquals(0, Double.compare(doubleVal, TbUtils.parseBigEndianHexToDouble(longValHex))); @@ -930,7 +931,13 @@ public class TbUtilsTest { @Test public void isDecimal_Test() { Assertions.assertEquals(10, TbUtils.isDecimal("4567039")); + Assertions.assertEquals(10, TbUtils.isDecimal("1.1428250947E8")); + Assertions.assertEquals(10, TbUtils.isDecimal("123.45")); + Assertions.assertEquals(10, TbUtils.isDecimal("-1.23E-4")); + Assertions.assertEquals(10, TbUtils.isDecimal("1E5")); Assertions.assertEquals(-1, TbUtils.isDecimal("C100110")); + Assertions.assertEquals(-1, TbUtils.isDecimal("abc")); + Assertions.assertEquals(-1, TbUtils.isDecimal(null)); } @Test diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java new file mode 100644 index 0000000000..327b477b6f --- /dev/null +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java @@ -0,0 +1,131 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.within; + +public class TbelCfTsRollingArgTest { + + private final long ts = System.currentTimeMillis(); + + private TbelCfTsRollingArg rollingArg; + + @BeforeEach + void setUp() { + rollingArg = new TbelCfTsRollingArg( + new TbTimeWindow(ts - 30000, ts - 10, 10), + List.of( + new TbelCfTsDoubleVal(ts - 10, Double.NaN), + new TbelCfTsDoubleVal(ts - 20, 2.0), + new TbelCfTsDoubleVal(ts - 30, 8.0), + new TbelCfTsDoubleVal(ts - 40, Double.NaN), + new TbelCfTsDoubleVal(ts - 50, 3.0), + new TbelCfTsDoubleVal(ts - 60, 9.0), + new TbelCfTsDoubleVal(ts - 70, Double.NaN) + ) + ); + } + + @Test + void testMax() { + assertThat(rollingArg.max()).isEqualTo(9.0); + assertThat(rollingArg.max(false)).isNaN(); + } + + @Test + void testMin() { + assertThat(rollingArg.min()).isEqualTo(2.0); + assertThat(rollingArg.min(false)).isNaN(); + } + + @Test + void testMean() { + assertThat(rollingArg.mean()).isEqualTo(5.5); + assertThat(rollingArg.mean(false)).isNaN(); + } + + @Test + void testStd() { + assertThat(rollingArg.std()).isCloseTo(3.0413812651491097, within(0.001)); + assertThat(rollingArg.std(false)).isNaN(); + } + + @Test + void testMedian() { + assertThat(rollingArg.median()).isEqualTo(5.5); + assertThat(rollingArg.median(false)).isNaN(); + } + + @Test + void testCount() { + assertThat(rollingArg.count()).isEqualTo(4); + assertThat(rollingArg.count(false)).isEqualTo(7); + } + + @Test + void testLast() { + assertThat(rollingArg.last()).isEqualTo(9.0); + assertThat(rollingArg.last(false)).isNaN(); + } + + @Test + void testFirst() { + assertThat(rollingArg.first()).isEqualTo(2.0); + assertThat(rollingArg.first(false)).isNaN(); + } + + @Test + void testFirstAndLastWhenOnlyNaNAndIgnoreNaNIsFalse() { + assertThat(rollingArg.first()).isEqualTo(2.0); + rollingArg = new TbelCfTsRollingArg( + new TbTimeWindow(ts - 30000, ts - 10, 10), + List.of( + new TbelCfTsDoubleVal(ts - 10, Double.NaN), + new TbelCfTsDoubleVal(ts - 40, Double.NaN), + new TbelCfTsDoubleVal(ts - 70, Double.NaN) + ) + ); + assertThatThrownBy(rollingArg::first).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::last).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + } + + @Test + void testSum() { + assertThat(rollingArg.sum()).isEqualTo(22.0); + assertThat(rollingArg.sum(false)).isNaN(); + } + + @Test + void testEmptyValues() { + rollingArg = new TbelCfTsRollingArg(new TbTimeWindow(0, 10, 10), List.of()); + assertThatThrownBy(rollingArg::sum).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::max).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::min).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::mean).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::std).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::median).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::first).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::last).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + } + +} \ No newline at end of file diff --git a/common/util/src/main/java/org/thingsboard/common/util/DebugModeUtil.java b/common/util/src/main/java/org/thingsboard/common/util/DebugModeUtil.java index d50ecfc5e0..c4b062a0d8 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/DebugModeUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/DebugModeUtil.java @@ -57,4 +57,14 @@ public final class DebugModeUtil { return debugSettings != null && nodeConnections != null && debugSettings.isFailuresEnabled() && nodeConnections.contains(TbNodeConnectionType.FAILURE); } } + + public static boolean isDebugFailuresAvailable(HasDebugSettings debugSettingsAware) { + if (isDebugAllAvailable(debugSettingsAware)) { + return true; + } else { + var debugSettings = debugSettingsAware.getDebugSettings(); + return debugSettings != null && debugSettings.isFailuresEnabled(); + } + } + } diff --git a/common/util/src/main/java/org/thingsboard/common/util/NoOpFutureCallback.java b/common/util/src/main/java/org/thingsboard/common/util/NoOpFutureCallback.java new file mode 100644 index 0000000000..176d949c95 --- /dev/null +++ b/common/util/src/main/java/org/thingsboard/common/util/NoOpFutureCallback.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.common.util; + +import com.google.common.util.concurrent.FutureCallback; + +public enum NoOpFutureCallback implements FutureCallback { + + INSTANCE; + + @Override + public void onSuccess(Object result) {} + + @Override + public void onFailure(Throwable t) {} + + @SuppressWarnings("unchecked") + public static FutureCallback instance() { + return (FutureCallback) INSTANCE; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/ResourceContainerDao.java b/dao/src/main/java/org/thingsboard/server/dao/ResourceContainerDao.java new file mode 100644 index 0000000000..6a952fd501 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/ResourceContainerDao.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.List; + +public interface ResourceContainerDao> { + + List findByTenantIdAndResourceLink(TenantId tenantId, String link, int limit); + + List findByResourceLink(String link, int limit); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index dfb708426f..36700ff59f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.asset; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.id.AssetId; @@ -103,6 +104,16 @@ public interface AssetDao extends Dao, TenantEntityDao, Exportable */ PageData findAssetInfosByTenantIdAndAssetProfileId(UUID tenantId, UUID assetProfileId, PageLink pageLink); + /** + * Find asset ids by tenantId, assetProfileId and page link. + * + * @param tenantId the tenantId + * @param assetProfileId the assetProfileId + * @param pageLink the page link + * @return the list of asset objects + */ + PageData findAssetIdsByTenantIdAndAssetProfileId(UUID tenantId, UUID assetProfileId, PageLink pageLink); + /** * Find assets by tenantId and assets Ids. * @@ -227,4 +238,6 @@ public interface AssetDao extends Dao, TenantEntityDao, Exportable PageData> getAllAssetTypes(PageLink pageLink); + PageData findProfileEntityIdInfos(PageLink pageLink); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index faed035fe3..21d5ea7f4e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -26,6 +26,7 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; @@ -177,8 +178,8 @@ public class BaseAssetService extends AbstractCachedEntityService findProfileEntityIdInfos(PageLink pageLink) { + log.trace("Executing findProfileEntityIdInfos, pageLink [{}]", pageLink); + validatePageLink(pageLink); + return assetDao.findProfileEntityIdInfos(pageLink); + } + + @Override + public PageData findAssetIdsByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink) { + log.trace("Executing findAssetIdsByTenantIdAndAssetProfileId, tenantId [{}], assetProfileId [{}]", tenantId, assetProfileId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(assetProfileId, id -> INCORRECT_ASSET_PROFILE_ID + id); + validatePageLink(pageLink); + return assetDao.findAssetIdsByTenantIdAndAssetProfileId(tenantId.getId(), assetProfileId.getId(), pageLink); + } + @Override public ListenableFuture> findAssetsByTenantIdAndIdsAsync(TenantId tenantId, List assetIds) { log.trace("Executing findAssetsByTenantIdAndIdsAsync, tenantId [{}], assetIds [{}]", tenantId, assetIds); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java new file mode 100644 index 0000000000..416bff6ea3 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -0,0 +1,209 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.cf; + +import com.google.common.util.concurrent.ListenableFuture; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; +import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; +import org.thingsboard.server.dao.exception.IncorrectParameterException; +import org.thingsboard.server.dao.service.DataValidator; + +import java.util.List; +import java.util.Optional; + +import static org.thingsboard.server.dao.service.Validator.validateId; +import static org.thingsboard.server.dao.service.Validator.validatePageLink; + +@Service("CalculatedFieldDaoService") +@Slf4j +@RequiredArgsConstructor +public class BaseCalculatedFieldService extends AbstractEntityService implements CalculatedFieldService { + + public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; + public static final String INCORRECT_CALCULATED_FIELD_ID = "Incorrect calculatedFieldId "; + public static final String INCORRECT_ENTITY_ID = "Incorrect entityId "; + + private final CalculatedFieldDao calculatedFieldDao; + private final CalculatedFieldLinkDao calculatedFieldLinkDao; + private final DataValidator calculatedFieldDataValidator; + private final DataValidator calculatedFieldLinkDataValidator; + + @Override + public CalculatedField save(CalculatedField calculatedField) { + CalculatedField oldCalculatedField = calculatedFieldDataValidator.validate(calculatedField, CalculatedField::getTenantId); + try { + TenantId tenantId = calculatedField.getTenantId(); + log.trace("Executing save calculated field, [{}]", calculatedField); + updateDebugSettings(tenantId, calculatedField, System.currentTimeMillis()); + CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); + createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); + eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedCalculatedField.getTenantId()).entityId(savedCalculatedField.getId()) + .entity(savedCalculatedField).oldEntity(oldCalculatedField).created(calculatedField.getId() == null).build()); + return savedCalculatedField; + } catch (Exception e) { + checkConstraintViolation(e, + "calculated_field_unq_key", "Calculated Field with such name is already in exists!", + "calculated_field_external_id_unq_key", "Calculated Field with such external id already exists!"); + throw e; + } + } + + @Override + public CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing findById, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedFieldId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); + return calculatedFieldDao.findById(tenantId, calculatedFieldId.getId()); + } + + @Override + public List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing findCalculatedFieldIdsByEntityId [{}]", entityId); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); + return calculatedFieldDao.findCalculatedFieldIdsByEntityId(tenantId, entityId); + } + + @Override + public List findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing findCalculatedFieldsByEntityId [{}]", entityId); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); + return calculatedFieldDao.findCalculatedFieldsByEntityId(tenantId, entityId); + } + + @Override + public PageData findAllCalculatedFields(PageLink pageLink) { + log.trace("Executing findAll, pageLink [{}]", pageLink); + validatePageLink(pageLink); + return calculatedFieldDao.findAll(pageLink); + } + + @Override + public PageData findAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink) { + log.trace("Executing findAllByEntityId, entityId [{}], pageLink [{}]", entityId, pageLink); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); + validatePageLink(pageLink); + return calculatedFieldDao.findAllByEntityId(tenantId, entityId, pageLink); + } + + @Override + public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); + deleteEntity(tenantId, calculatedFieldId, false); + } + + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + CalculatedField calculatedField = calculatedFieldDao.findById(tenantId, id.getId()); + if (calculatedField == null) { + if (force) { + return; + } else { + throw new IncorrectParameterException("Unable to delete non-existent calculated field."); + } + } + deleteCalculatedField(tenantId, calculatedField); + } + + private void deleteCalculatedField(TenantId tenantId, CalculatedField calculatedField) { + log.trace("Executing deleteCalculatedField, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedField.getId()); + calculatedFieldDao.removeById(tenantId, calculatedField.getUuidId()); + eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(calculatedField.getId()).entity(calculatedField).build()); + } + + @Override + public int deleteAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing deleteAllCalculatedFieldsByEntityId, tenantId [{}], entityId [{}]", tenantId, entityId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); + List calculatedFields = calculatedFieldDao.removeAllByEntityId(tenantId, entityId); + return calculatedFields.size(); + } + + @Override + public CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink) { + calculatedFieldLinkDataValidator.validate(calculatedFieldLink, CalculatedFieldLink::getTenantId); + log.trace("Executing save calculated field link, [{}]", calculatedFieldLink); + return calculatedFieldLinkDao.save(tenantId, calculatedFieldLink); + } + + @Override + public CalculatedFieldLink findCalculatedFieldLinkById(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId) { + log.trace("Executing findCalculatedFieldLinkById, tenantId [{}], calculatedFieldLinkId [{}]", tenantId, calculatedFieldLinkId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(calculatedFieldLinkId, id -> "Incorrect calculatedFieldLinkId " + id); + return calculatedFieldLinkDao.findById(tenantId, calculatedFieldLinkId.getId()); + } + + @Override + public List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing findAllCalculatedFieldLinksById, calculatedFieldId [{}]", calculatedFieldId); + return calculatedFieldLinkDao.findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId); + } + + @Override + public List findAllCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing findAllCalculatedFieldLinksByEntityId, entityId [{}]", entityId); + return calculatedFieldLinkDao.findCalculatedFieldLinksByEntityId(tenantId, entityId); + } + + @Override + public PageData findAllCalculatedFieldLinks(PageLink pageLink) { + log.trace("Executing findAllCalculatedFieldLinks, pageLink [{}]", pageLink); + validatePageLink(pageLink); + return calculatedFieldLinkDao.findAll(pageLink); + } + + @Override + public boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId) { + return calculatedFieldDao.findAllByTenantId(tenantId).stream() + .filter(calculatedField -> !referencedEntityId.equals(calculatedField.getEntityId())) + .map(CalculatedField::getConfiguration) + .map(CalculatedFieldConfiguration::getReferencedEntities) + .anyMatch(referencedEntities -> referencedEntities.contains(referencedEntityId)); + } + + @Override + public Optional> findEntity(TenantId tenantId, EntityId entityId) { + return Optional.ofNullable(findById(tenantId, new CalculatedFieldId(entityId.getId()))); + } + + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD; + } + + private void createOrUpdateCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { + List links = calculatedField.getConfiguration().buildCalculatedFieldLinks(tenantId, calculatedField.getEntityId(), calculatedField.getId()); + links.forEach(link -> saveCalculatedFieldLink(tenantId, link)); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java new file mode 100644 index 0000000000..a966977968 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.cf; + +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.Dao; + +import java.util.List; + +public interface CalculatedFieldDao extends Dao { + + List findAllByTenantId(TenantId tenantId); + + List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId); + + List findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + + List findAll(); + + PageData findAll(PageLink pageLink); + + PageData findAllByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink); + + List removeAllByEntityId(TenantId tenantId, EntityId entityId); + + long countCFByEntityId(TenantId tenantId, EntityId entityId); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java new file mode 100644 index 0000000000..8b4a5e7086 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.cf; + +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.Dao; + +import java.util.List; + +public interface CalculatedFieldLinkDao extends Dao { + + List findCalculatedFieldLinksByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + List findCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + + List findAll(); + + PageData findAll(PageLink pageLink); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java index 61b2066108..4b05cdbc75 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java @@ -184,6 +184,10 @@ public class CustomerServiceImpl extends AbstractCachedEntityService, ImageContainerDao { +public interface DashboardInfoDao extends Dao, ImageContainerDao, ResourceContainerDao { /** * Find dashboards by tenantId and page link. diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index a148b79369..efc57119eb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -22,7 +22,10 @@ import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.DeviceInfoFilter; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; @@ -85,6 +88,16 @@ public interface DeviceDao extends Dao, TenantEntityDao, Exporta */ PageData findDevicesByTenantIdAndType(UUID tenantId, String type, PageLink pageLink); + /** + * Find device ids by tenantId, type and page link. + * + * @param tenantId the tenantId + * @param deviceProfileId the deviceProfileId + * @param pageLink the page link + * @return the list of device objects + */ + PageData findDeviceIdsByTenantIdAndDeviceProfileId(UUID tenantId, UUID deviceProfileId, PageLink pageLink); + PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(UUID tenantId, UUID deviceProfileId, OtaPackageType type, @@ -218,5 +231,8 @@ public interface DeviceDao extends Dao, TenantEntityDao, Exporta PageData findDeviceIdInfos(PageLink pageLink); + PageData findProfileEntityIdInfos(PageLink pageLink); + PageData findDeviceInfosByFilter(DeviceInfoFilter filter, PageLink pageLink); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index 6a601babcb..2c890082a0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.audit.ActionType; @@ -337,8 +338,8 @@ public class DeviceServiceImpl extends CachedVersionedEntityService findProfileEntityIdInfos(PageLink pageLink) { + log.trace("Executing findProfileEntityIdInfos, pageLink [{}]", pageLink); + validatePageLink(pageLink); + return deviceDao.findProfileEntityIdInfos(pageLink); + } + @Override public PageData findDevicesByTenantIdAndType(TenantId tenantId, String type, PageLink pageLink) { log.trace("Executing findDevicesByTenantIdAndType, tenantId [{}], type [{}], pageLink [{}]", tenantId, type, pageLink); @@ -395,6 +403,15 @@ public class DeviceServiceImpl extends CachedVersionedEntityService findDeviceIdsByTenantIdAndDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId, PageLink pageLink) { + log.trace("Executing findDeviceIdsByTenantIdAndType, tenantId [{}], deviceProfileId [{}], pageLink [{}]", tenantId, deviceProfileId, pageLink); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(deviceProfileId, id -> INCORRECT_DEVICE_PROFILE_ID + id); + validatePageLink(pageLink); + return deviceDao.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId.getId(), deviceProfileId.getId(), pageLink); + } + @Override public PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java index 75fc713be3..7560c7fb76 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.exception.DataValidationException; @@ -66,6 +67,10 @@ public abstract class AbstractEntityService { @Autowired protected EntityViewService entityViewService; + @Lazy + @Autowired + protected CalculatedFieldService calculatedFieldService; + @Lazy @Autowired(required = false) protected EdgeService edgeService; diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 3e398f9746..6e32901517 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -178,6 +178,11 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe return fetchAndConvert(tenantId, entityId, this::getNameLabelAndCustomerDetails); } + @Override + public Optional> fetchEntity(TenantId tenantId, EntityId entityId) { + return fetchAndConvert(tenantId, entityId, Function.identity()); + } + private Optional fetchAndConvert(TenantId tenantId, EntityId entityId, Function, T> converter) { EntityDaoService entityDaoService = entityServiceRegistry.getServiceByEntityType(entityId.getEntityType()); Optional> entityOpt = entityDaoService.findEntity(tenantId, entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java b/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java index a795bddffa..9c50ad621f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java @@ -43,6 +43,9 @@ public class DefaultEntityServiceRegistry implements EntityServiceRegistry { if (EntityType.RULE_CHAIN.equals(entityType)) { entityDaoServicesMap.put(EntityType.RULE_NODE, entityDaoService); } + if (EntityType.CALCULATED_FIELD.equals(entityType)) { + entityDaoServicesMap.put(EntityType.CALCULATED_FIELD_LINK, entityDaoService); + } }); log.debug("Initialized EntityServiceRegistry total [{}] entries", entityDaoServicesMap.size()); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java index 0f8e174c7e..d81d8eb0f3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java @@ -23,6 +23,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; @@ -88,6 +89,11 @@ public class BaseEventService implements EventService { ErrorEvent eEvent = (ErrorEvent) event; truncateField(eEvent, ErrorEvent::getError, ErrorEvent::setError); break; + case DEBUG_CALCULATED_FIELD: + CalculatedFieldDebugEvent cfEvent = (CalculatedFieldDebugEvent) event; + truncateField(cfEvent, CalculatedFieldDebugEvent::getArguments, CalculatedFieldDebugEvent::setArguments); + truncateField(cfEvent, CalculatedFieldDebugEvent::getResult, CalculatedFieldDebugEvent::setResult); + truncateField(cfEvent, CalculatedFieldDebugEvent::getError, CalculatedFieldDebugEvent::setError); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java index 120287fb83..1ca2973936 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java @@ -75,6 +75,7 @@ public class CleanUpService { submitTask(HousekeeperTask.deleteTelemetry(tenantId, entityId)); submitTask(HousekeeperTask.deleteEvents(tenantId, entityId)); submitTask(HousekeeperTask.deleteAlarms(tenantId, entityId)); + submitTask(HousekeeperTask.deleteCalculatedFields(tenantId, entityId)); } public void removeTenantEntities(TenantId tenantId, EntityType... entityTypes) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index aaf9659f1b..e2b21ed59b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -376,6 +376,7 @@ public class ModelConstants { public static final String STATS_EVENT_TABLE_NAME = "stats_event"; public static final String RULE_NODE_DEBUG_EVENT_TABLE_NAME = "rule_node_debug_event"; public static final String RULE_CHAIN_DEBUG_EVENT_TABLE_NAME = "rule_chain_debug_event"; + public static final String CALCULATED_FIELD_DEBUG_EVENT_TABLE_NAME = "cf_debug_event"; public static final String EVENT_TENANT_ID_PROPERTY = TENANT_ID_PROPERTY; public static final String EVENT_SERVICE_ID_PROPERTY = "service_id"; @@ -400,6 +401,10 @@ public class ModelConstants { public static final String EVENT_METADATA_COLUMN_NAME = "e_metadata"; public static final String EVENT_MESSAGE_COLUMN_NAME = "e_message"; + public static final String EVENT_CALCULATED_FIELD_ID_COLUMN_NAME = "cf_id"; + public static final String EVENT_CALCULATED_FIELD_ARGUMENTS_COLUMN_NAME = "e_args"; + public static final String EVENT_CALCULATED_FIELD_RESULT_COLUMN_NAME = "e_result"; + public static final String DEBUG_MODE = "debug_mode"; public static final String DEBUG_SETTINGS = "debug_settings"; public static final String SINGLETON_MODE = "singleton_mode"; @@ -712,6 +717,29 @@ public class ModelConstants { public static final String QR_CODE_SETTINGS_BUNDLE_ID_PROPERTY = "mobile_app_bundle_id"; public static final String QR_CODE_SETTINGS_CONFIG_PROPERTY = "qr_code_config"; + /** + * Calculated fields constants. + */ + public static final String CALCULATED_FIELD_TABLE_NAME = "calculated_field"; + public static final String CALCULATED_FIELD_TENANT_ID_COLUMN = TENANT_ID_COLUMN; + public static final String CALCULATED_FIELD_ENTITY_TYPE = ENTITY_TYPE_COLUMN; + public static final String CALCULATED_FIELD_ENTITY_ID = ENTITY_ID_COLUMN; + public static final String CALCULATED_FIELD_TYPE = "type"; + public static final String CALCULATED_FIELD_NAME = "name"; + public static final String CALCULATED_FIELD_CONFIGURATION_VERSION = "configuration_version"; + public static final String CALCULATED_FIELD_CONFIGURATION = "configuration"; + public static final String CALCULATED_FIELD_VERSION = "version"; + public static final String CALCULATED_FIELD_EXTERNAL_ID = "external_id"; + + /** + * Calculated field links constants. + */ + public static final String CALCULATED_FIELD_LINK_TABLE_NAME = "calculated_field_link"; + public static final String CALCULATED_FIELD_LINK_TENANT_ID_COLUMN = TENANT_ID_COLUMN; + public static final String CALCULATED_FIELD_LINK_ENTITY_TYPE = ENTITY_TYPE_COLUMN; + public static final String CALCULATED_FIELD_LINK_ENTITY_ID = ENTITY_ID_COLUMN; + public static final String CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID = "calculated_field_id"; + protected static final String[] NONE_AGGREGATION_COLUMNS = new String[]{LONG_VALUE_COLUMN, DOUBLE_VALUE_COLUMN, BOOLEAN_VALUE_COLUMN, STRING_VALUE_COLUMN, JSON_VALUE_COLUMN, KEY_COLUMN, TS_COLUMN}; protected static final String[] COUNT_AGGREGATION_COLUMNS = new String[]{count(LONG_VALUE_COLUMN), count(DOUBLE_VALUE_COLUMN), count(BOOLEAN_VALUE_COLUMN), count(STRING_VALUE_COLUMN), count(JSON_VALUE_COLUMN), max(TS_COLUMN)}; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ApiUsageStateEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ApiUsageStateEntity.java index 6caacd0ffb..40f7442047 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ApiUsageStateEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ApiUsageStateEntity.java @@ -28,7 +28,7 @@ import org.thingsboard.server.common.data.id.ApiUsageStateId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.BaseEntity; -import org.thingsboard.server.dao.model.BaseSqlEntity; +import org.thingsboard.server.dao.model.BaseVersionedEntity; import org.thingsboard.server.dao.model.ModelConstants; import java.util.UUID; @@ -40,7 +40,7 @@ import java.util.UUID; @EqualsAndHashCode(callSuper = true) @Entity @Table(name = ModelConstants.API_USAGE_STATE_TABLE_NAME) -public class ApiUsageStateEntity extends BaseSqlEntity implements BaseEntity { +public class ApiUsageStateEntity extends BaseVersionedEntity implements BaseEntity { @Column(name = ModelConstants.API_USAGE_STATE_TENANT_ID_COLUMN) private UUID tenantId; @@ -77,10 +77,7 @@ public class ApiUsageStateEntity extends BaseSqlEntity implements } public ApiUsageStateEntity(ApiUsageState ur) { - if (ur.getId() != null) { - this.setUuid(ur.getId().getId()); - } - this.setCreatedTime(ur.getCreatedTime()); + super(ur); if (ur.getTenantId() != null) { this.tenantId = ur.getTenantId().getId(); } @@ -116,6 +113,7 @@ public class ApiUsageStateEntity extends BaseSqlEntity implements ur.setEmailExecState(emailExecState); ur.setSmsExecState(smsExecState); ur.setAlarmExecState(alarmExecState); + ur.setVersion(version); return ur; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldDebugEventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldDebugEventEntity.java new file mode 100644 index 0000000000..cf771238b9 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldDebugEventEntity.java @@ -0,0 +1,104 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Table; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; + +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_DEBUG_EVENT_TABLE_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_CALCULATED_FIELD_ARGUMENTS_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_CALCULATED_FIELD_ID_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_CALCULATED_FIELD_RESULT_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_ID_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_TYPE_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERROR_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MSG_ID_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MSG_TYPE_COLUMN_NAME; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = CALCULATED_FIELD_DEBUG_EVENT_TABLE_NAME) +@NoArgsConstructor +public class CalculatedFieldDebugEventEntity extends EventEntity implements BaseEntity { + + @Column(name = EVENT_CALCULATED_FIELD_ID_COLUMN_NAME) + private UUID calculatedFieldId; + @Column(name = EVENT_ENTITY_ID_COLUMN_NAME) + private UUID eventEntityId; + @Column(name = EVENT_ENTITY_TYPE_COLUMN_NAME) + private String eventEntityType; + @Column(name = EVENT_MSG_ID_COLUMN_NAME) + private UUID msgId; + @Column(name = EVENT_MSG_TYPE_COLUMN_NAME) + private String msgType; + @Column(name = EVENT_CALCULATED_FIELD_ARGUMENTS_COLUMN_NAME) + private String arguments; + @Column(name = EVENT_CALCULATED_FIELD_RESULT_COLUMN_NAME) + private String result; + @Column(name = EVENT_ERROR_COLUMN_NAME) + private String error; + + public CalculatedFieldDebugEventEntity(CalculatedFieldDebugEvent event) { + super(event); + if (event.getCalculatedFieldId() != null) { + this.calculatedFieldId = event.getCalculatedFieldId().getId(); + } + if (event.getEventEntity() != null) { + this.eventEntityId = event.getEventEntity().getId(); + this.eventEntityType = event.getEventEntity().getEntityType().name(); + } + this.msgId = event.getMsgId(); + this.msgType = event.getMsgType(); + this.arguments = event.getArguments(); + this.result = event.getResult(); + this.error = event.getError(); + } + + @Override + public CalculatedFieldDebugEvent toData() { + var builder = CalculatedFieldDebugEvent.builder() + .id(id) + .tenantId(TenantId.fromUUID(tenantId)) + .ts(ts) + .serviceId(serviceId) + .entityId(entityId) + .msgId(msgId) + .msgType(msgType) + .arguments(arguments) + .result(result) + .error(error); + if (calculatedFieldId != null) { + builder.calculatedFieldId(new CalculatedFieldId(calculatedFieldId)); + } + if (eventEntityId != null) { + builder.eventEntity(EntityIdFactory.getByTypeAndUuid(eventEntityType, eventEntityId)); + } + return builder.build(); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java new file mode 100644 index 0000000000..349091a1ae --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -0,0 +1,127 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import com.fasterxml.jackson.databind.JsonNode; +import jakarta.persistence.Column; +import jakarta.persistence.Convert; +import jakarta.persistence.Entity; +import jakarta.persistence.Table; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; +import org.thingsboard.server.dao.model.BaseVersionedEntity; +import org.thingsboard.server.dao.util.mapping.JsonConverter; + +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION_VERSION; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_ID; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_TYPE; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_EXTERNAL_ID; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TABLE_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TENANT_ID_COLUMN; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TYPE; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_VERSION; +import static org.thingsboard.server.dao.model.ModelConstants.DEBUG_SETTINGS; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = CALCULATED_FIELD_TABLE_NAME) +public class CalculatedFieldEntity extends BaseVersionedEntity implements BaseEntity { + + @Column(name = CALCULATED_FIELD_TENANT_ID_COLUMN) + private UUID tenantId; + + @Column(name = CALCULATED_FIELD_ENTITY_TYPE) + private String entityType; + + @Column(name = CALCULATED_FIELD_ENTITY_ID) + private UUID entityId; + + @Column(name = CALCULATED_FIELD_TYPE) + private String type; + + @Column(name = CALCULATED_FIELD_NAME) + private String name; + + @Column(name = CALCULATED_FIELD_CONFIGURATION_VERSION) + private int configurationVersion; + + @Convert(converter = JsonConverter.class) + @Column(name = CALCULATED_FIELD_CONFIGURATION) + private JsonNode configuration; + + @Column(name = CALCULATED_FIELD_VERSION) + private Long version; + + @Column(name = DEBUG_SETTINGS) + private String debugSettings; + + @Column(name = CALCULATED_FIELD_EXTERNAL_ID) + private UUID externalId; + + public CalculatedFieldEntity() { + super(); + } + + public CalculatedFieldEntity(CalculatedField calculatedField) { + this.setUuid(calculatedField.getUuidId()); + this.createdTime = calculatedField.getCreatedTime(); + this.tenantId = calculatedField.getTenantId().getId(); + this.entityType = calculatedField.getEntityId().getEntityType().name(); + this.entityId = calculatedField.getEntityId().getId(); + this.type = calculatedField.getType().name(); + this.name = calculatedField.getName(); + this.configurationVersion = calculatedField.getConfigurationVersion(); + this.configuration = JacksonUtil.valueToTree(calculatedField.getConfiguration()); + this.version = calculatedField.getVersion(); + this.debugSettings = JacksonUtil.toString(calculatedField.getDebugSettings()); + if (calculatedField.getExternalId() != null) { + this.externalId = calculatedField.getExternalId().getId(); + } + } + + @Override + public CalculatedField toData() { + CalculatedField calculatedField = new CalculatedField(new CalculatedFieldId(id)); + calculatedField.setCreatedTime(createdTime); + calculatedField.setTenantId(TenantId.fromUUID(tenantId)); + calculatedField.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedField.setType(CalculatedFieldType.valueOf(type)); + calculatedField.setName(name); + calculatedField.setConfigurationVersion(configurationVersion); + calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); + calculatedField.setVersion(version); + calculatedField.setDebugSettings(JacksonUtil.fromString(debugSettings, DebugSettings.class)); + if (externalId != null) { + calculatedField.setExternalId(new CalculatedFieldId(externalId)); + } + return calculatedField; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java new file mode 100644 index 0000000000..0f2a6455ec --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -0,0 +1,79 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Table; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; +import org.thingsboard.server.dao.model.BaseSqlEntity; + +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_ID; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_TYPE; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_TABLE_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_TENANT_ID_COLUMN; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = CALCULATED_FIELD_LINK_TABLE_NAME) +public class CalculatedFieldLinkEntity extends BaseSqlEntity implements BaseEntity { + + @Column(name = CALCULATED_FIELD_LINK_TENANT_ID_COLUMN) + private UUID tenantId; + + @Column(name = CALCULATED_FIELD_LINK_ENTITY_TYPE) + private String entityType; + + @Column(name = CALCULATED_FIELD_LINK_ENTITY_ID) + private UUID entityId; + + @Column(name = CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID) + private UUID calculatedFieldId; + + public CalculatedFieldLinkEntity() { + super(); + } + + public CalculatedFieldLinkEntity(CalculatedFieldLink calculatedFieldLink) { + super(calculatedFieldLink); + this.tenantId = calculatedFieldLink.getTenantId().getId(); + this.entityType = calculatedFieldLink.getEntityId().getEntityType().name(); + this.entityId = calculatedFieldLink.getEntityId().getId(); + this.calculatedFieldId = calculatedFieldLink.getCalculatedFieldId().getId(); + } + + @Override + public CalculatedFieldLink toData() { + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(new CalculatedFieldLinkId(id)); + calculatedFieldLink.setCreatedTime(createdTime); + calculatedFieldLink.setTenantId(TenantId.fromUUID(tenantId)); + calculatedFieldLink.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedFieldLink.setCalculatedFieldId(new CalculatedFieldId(calculatedFieldId)); + return calculatedFieldLink; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java index f8fe55cf02..e91c1939f6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java @@ -75,8 +75,6 @@ import static org.thingsboard.server.common.data.StringUtils.isNotEmpty; @Slf4j public class BaseImageService extends BaseResourceService implements ImageService { - private static final int MAX_ENTITIES_TO_FIND = 10; - public static Map DASHBOARD_BASE64_MAPPING = new HashMap<>(); public static Map WIDGET_TYPE_BASE64_MAPPING = new HashMap<>(); @@ -107,19 +105,15 @@ public class BaseImageService extends BaseResourceService implements ImageServic private final AssetProfileDao assetProfileDao; private final DeviceProfileDao deviceProfileDao; private final WidgetsBundleDao widgetsBundleDao; - private final WidgetTypeDao widgetTypeDao; - private final DashboardInfoDao dashboardInfoDao; private final Map> imageContainerDaoMap = new HashMap<>(); public BaseImageService(TbResourceDao resourceDao, TbResourceInfoDao resourceInfoDao, ResourceDataValidator resourceValidator, AssetProfileDao assetProfileDao, DeviceProfileDao deviceProfileDao, WidgetsBundleDao widgetsBundleDao, WidgetTypeDao widgetTypeDao, DashboardInfoDao dashboardInfoDao) { - super(resourceDao, resourceInfoDao, resourceValidator); + super(resourceDao, resourceInfoDao, resourceValidator, widgetTypeDao, dashboardInfoDao); this.assetProfileDao = assetProfileDao; this.deviceProfileDao = deviceProfileDao; this.widgetsBundleDao = widgetsBundleDao; - this.widgetTypeDao = widgetTypeDao; - this.dashboardInfoDao = dashboardInfoDao; } @PostConstruct @@ -131,7 +125,6 @@ public class BaseImageService extends BaseResourceService implements ImageServic imageContainerDaoMap.put(EntityType.DASHBOARD, dashboardInfoDao); } - @Override @SneakyThrows public TbResourceInfo saveImage(TbResource image) { @@ -311,7 +304,8 @@ public class BaseImageService extends BaseResourceService implements ImageServic } } if (success) { - deleteResource(tenantId, imageId, force); + success = deleteResource(tenantId, imageId, true) + .isSuccess(); } return result.success(success).build(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java index 76aefdbb93..bf73c59708 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.hash.Hashing; import com.google.common.util.concurrent.ListenableFuture; +import jakarta.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -39,6 +40,7 @@ import org.thingsboard.server.common.data.ResourceExportData; import org.thingsboard.server.common.data.ResourceSubType; import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TbResourceDeleteResult; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.TbResourceInfoFilter; import org.thingsboard.server.common.data.id.EntityId; @@ -48,6 +50,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; +import org.thingsboard.server.dao.ResourceContainerDao; +import org.thingsboard.server.dao.dashboard.DashboardInfoDao; import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; @@ -55,6 +59,7 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; import org.thingsboard.server.dao.service.validator.ResourceDataValidator; +import org.thingsboard.server.dao.widget.WidgetTypeDao; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -85,6 +90,17 @@ public class BaseResourceService extends AbstractCachedEntityService> resourceContainerDaoMap = new HashMap<>(); + protected static final int MAX_ENTITIES_TO_FIND = 10; + + @PostConstruct + public void init() { + resourceContainerDaoMap.put(EntityType.WIDGET_TYPE, widgetTypeDao); + resourceContainerDaoMap.put(EntityType.DASHBOARD, dashboardInfoDao); + } + @Autowired @Lazy private ImageService imageService; @@ -313,23 +329,45 @@ public class BaseResourceService extends AbstractCachedEntityService INCORRECT_RESOURCE_ID + id); TbResourceInfo resource = findResourceInfoById(tenantId, resourceId); + boolean success = true; + var result = TbResourceDeleteResult.builder(); + if (resource == null) { - return; + if (!force) { + success = false; + } + return result.success(success).build(); } + if (!force) { - resourceValidator.validateDelete(tenantId, resource); + if (resource.getResourceType() == ResourceType.JS_MODULE) { + var link = resource.getLink(); + Map>> affectedEntities = new HashMap<>(); + + resourceContainerDaoMap.forEach((entityType, resourceContainerDao) -> { + var entities = tenantId.isSysTenantId() ? resourceContainerDao.findByResourceLink(link, MAX_ENTITIES_TO_FIND) : + resourceContainerDao.findByTenantIdAndResourceLink(tenantId, link, MAX_ENTITIES_TO_FIND); + if (!entities.isEmpty()) { + affectedEntities.put(entityType.name(), entities); + } + }); + + if (!affectedEntities.isEmpty()) { + success = false; + result.references(affectedEntities); + } + } } - resourceDao.removeById(tenantId, resourceId.getId()); - eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entity(resource).entityId(resourceId).build()); + if (success) { + resourceDao.removeById(tenantId, resourceId.getId()); + eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entity(resource).entityId(resourceId).build()); + } + + return result.success(success).build(); } @Override @@ -666,7 +704,7 @@ public class BaseResourceService extends AbstractCachedEntityService { + + @Autowired + private CalculatedFieldDao calculatedFieldDao; + + @Autowired + private ApiLimitService apiLimitService; + + @Override + protected void validateCreate(TenantId tenantId, CalculatedField calculatedField) { + validateNumberOfCFsPerEntity(tenantId, calculatedField.getEntityId()); + validateNumberOfArgumentsPerCF(tenantId, calculatedField); + } + + @Override + protected CalculatedField validateUpdate(TenantId tenantId, CalculatedField calculatedField) { + CalculatedField old = calculatedFieldDao.findById(calculatedField.getTenantId(), calculatedField.getId().getId()); + if (old == null) { + throw new DataValidationException("Can't update non existing calculated field!"); + } + validateNumberOfArgumentsPerCF(tenantId, calculatedField); + return old; + } + + private void validateNumberOfCFsPerEntity(TenantId tenantId, EntityId entityId) { + long maxCFsPerEntity = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxCalculatedFieldsPerEntity); + if (maxCFsPerEntity <= 0) { + return; + } + if (calculatedFieldDao.countCFByEntityId(tenantId, entityId) >= maxCFsPerEntity) { + throw new DataValidationException("Calculated fields per entity limit reached!"); + } + } + + private void validateNumberOfArgumentsPerCF(TenantId tenantId, CalculatedField calculatedField) { + long maxArgumentsPerCF = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxArgumentsPerCF); + if (maxArgumentsPerCF <= 0) { + return; + } + if (calculatedField.getConfiguration().getArguments().size() > maxArgumentsPerCF) { + throw new DataValidationException("Calculated field arguments limit reached!"); + } + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java new file mode 100644 index 0000000000..aaba200c92 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.validator; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.cf.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.service.DataValidator; + +@Component +public class CalculatedFieldLinkDataValidator extends DataValidator { + + @Autowired + private CalculatedFieldLinkDao calculatedFieldLinkDao; + + @Override + protected CalculatedFieldLink validateUpdate(TenantId tenantId, CalculatedFieldLink calculatedFieldLink) { + CalculatedFieldLink old = calculatedFieldLinkDao.findById(calculatedFieldLink.getTenantId(), calculatedFieldLink.getId().getId()); + if (old == null) { + throw new DataValidationException("Can't update non existing calculated field link!"); + } + return old; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java index 2f49e7008b..f9f03adeff 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java @@ -21,7 +21,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.TbResource; -import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.id.TbResourceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; @@ -30,9 +29,6 @@ import org.thingsboard.server.dao.resource.TbResourceDao; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.dao.widget.WidgetTypeDao; - -import java.util.List; import static org.thingsboard.server.common.data.EntityType.TB_RESOURCE; @@ -42,9 +38,6 @@ public class ResourceDataValidator extends DataValidator { @Autowired private TbResourceDao resourceDao; - @Autowired - private WidgetTypeDao widgetTypeDao; - @Autowired private TenantService tenantService; @@ -111,12 +104,4 @@ public class ResourceDataValidator extends DataValidator { validateMaxSumDataSizePerTenant(tenantId, resourceDao, maxSumResourcesDataInBytes, dataSize, TB_RESOURCE); } } - - public void validateDelete(TenantId tenantId, TbResourceInfo resourceInfo) { - List widgets = widgetTypeDao.findWidgetTypesNamesByTenantIdAndResourceLink(tenantId.getId(), resourceInfo.getLink()); - if (!widgets.isEmpty()) { - throw new DataValidationException("Following widget types use this resource: " + String.join(", ", widgets)); - } - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java index 0f2550b7a2..05577b68e1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java @@ -88,10 +88,10 @@ public abstract class JpaAbstractDao, D> boolean flushed = false; EntityManager entityManager = getEntityManager(); if (isNew) { + entityManager.persist(entity); if (entity instanceof HasVersion versionedEntity) { versionedEntity.setVersion(1L); } - entityManager.persist(entity); } else { if (entity instanceof HasVersion versionedEntity) { if (versionedEntity.getVersion() == null) { @@ -106,23 +106,25 @@ public abstract class JpaAbstractDao, D> } } versionedEntity = entityManager.merge(versionedEntity); + entity = (E) versionedEntity; /* * by default, Hibernate doesn't issue an update query and thus version increment * if the entity was not modified. to bypass this and always increment the version, we do it manually * */ versionedEntity.setVersion(versionedEntity.getVersion() + 1); - /* - * flushing and then removing the entity from the persistence context so that it is not affected - * by next flushes (e.g. when a transaction is committed) to avoid double version increment - * */ - entityManager.flush(); - entityManager.detach(versionedEntity); - flushed = true; - entity = (E) versionedEntity; } else { entity = entityManager.merge(entity); } } + if (entity instanceof HasVersion versionedEntity) { + /* + * flushing and then removing the entity from the persistence context so that it is not affected + * by next flushes (e.g. when a transaction is committed) to avoid double version increment + * */ + entityManager.flush(); + entityManager.detach(versionedEntity); + flushed = true; + } if (flush && !flushed) { entityManager.flush(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java index 8ba9ea4f50..e475864684 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java @@ -141,6 +141,15 @@ public interface AssetRepository extends JpaRepository, Expor @Param("textSearch") String textSearch, Pageable pageable); + @Query("SELECT a.id FROM AssetEntity a " + + "WHERE a.tenantId = :tenantId " + + "AND a.assetProfileId = :assetProfileId " + + "AND (:textSearch IS NULL OR ilike(a.type, CONCAT('%', :textSearch, '%')) = true) ") + Page findAssetIdsByTenantIdAndAssetProfileId(@Param("tenantId") UUID tenantId, + @Param("assetProfileId") UUID assetProfileId, + @Param("textSearch") String textSearch, + Pageable pageable); + @Query("SELECT a FROM AssetEntity a WHERE a.tenantId = :tenantId " + "AND a.customerId = :customerId AND a.type = :type " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index 4b3db1f49b..c61d894de5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -23,6 +23,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.edqs.fields.AssetFields; @@ -37,12 +38,15 @@ import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.model.sql.AssetEntity; import org.thingsboard.server.dao.model.sql.AssetInfoEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; +import org.thingsboard.server.dao.sql.device.NativeAssetRepository; +import org.thingsboard.server.dao.sql.device.NativeDeviceRepository; import org.thingsboard.server.dao.util.SqlDao; import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; import static org.thingsboard.server.dao.DaoUtil.convertTenantEntityInfosToDto; @@ -57,6 +61,9 @@ public class JpaAssetDao extends JpaAbstractDao implements A @Autowired private AssetRepository assetRepository; + @Autowired + private NativeAssetRepository nativeAssetRepository; + @Autowired private AssetProfileRepository assetProfileRepository; @@ -161,6 +168,16 @@ public class JpaAssetDao extends JpaAbstractDao implements A DaoUtil.toPageable(pageLink, AssetInfoEntity.assetInfoColumnMap))); } + @Override + public PageData findAssetIdsByTenantIdAndAssetProfileId(UUID tenantId, UUID assetProfileId, PageLink pageLink) { + return DaoUtil.pageToPageData(assetRepository.findAssetIdsByTenantIdAndAssetProfileId( + tenantId, + assetProfileId, + pageLink.getTextSearch(), + DaoUtil.toPageable(pageLink))) + .mapData(AssetId::new); + } + @Override public PageData findAssetsByTenantIdAndCustomerIdAndType(UUID tenantId, UUID customerId, String type, PageLink pageLink) { return DaoUtil.toPageData(assetRepository @@ -243,6 +260,12 @@ public class JpaAssetDao extends JpaAbstractDao implements A DaoUtil.toPageable(pageLink, Arrays.asList(new SortOrder("tenantId"), new SortOrder("type"))))); } + @Override + public PageData findProfileEntityIdInfos(PageLink pageLink) { + log.debug("Find profile device id infos by pageLink [{}]", pageLink); + return nativeAssetRepository.findProfileEntityIdInfos(DaoUtil.toPageable(pageLink)); + } + @Override public Long countByTenantId(TenantId tenantId) { return assetRepository.countByTenantId(tenantId.getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java new file mode 100644 index 0000000000..584a3b5199 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.cf; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; + +import java.util.List; +import java.util.UUID; + +public interface CalculatedFieldLinkRepository extends JpaRepository { + + List findAllByTenantIdAndCalculatedFieldId(UUID tenantId, UUID calculatedFieldId); + + List findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java new file mode 100644 index 0000000000..0f48f3b00d --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.cf; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; + +import java.util.List; +import java.util.UUID; + +public interface CalculatedFieldRepository extends JpaRepository { + + boolean existsByTenantIdAndEntityId(UUID tenantId, UUID entityId); + + List findCalculatedFieldIdsByTenantIdAndEntityId(UUID tenantId, UUID entityId); + + List findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + + Page findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId, Pageable pageable); + + List findAllByTenantId(UUID tenantId); + + List removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + + long countByTenantIdAndEntityId(UUID tenantId, UUID entityId); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java new file mode 100644 index 0000000000..fae3468f1e --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -0,0 +1,137 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.cf; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Pageable; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; + +@RequiredArgsConstructor +@Repository +@Slf4j +public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedFieldRepository { + + private final String CF_COUNT_QUERY = "SELECT count(id) FROM calculated_field;"; + private final String CF_QUERY = "SELECT * FROM calculated_field ORDER BY created_time ASC LIMIT %s OFFSET %s"; + + private final String CFL_COUNT_QUERY = "SELECT count(id) FROM calculated_field_link;"; + private final String CFL_QUERY = "SELECT * FROM calculated_field_link ORDER BY created_time ASC LIMIT %s OFFSET %s"; + + private final NamedParameterJdbcTemplate jdbcTemplate; + private final TransactionTemplate transactionTemplate; + + @Override + public PageData findCalculatedFields(Pageable pageable) { + return transactionTemplate.execute(status -> { + long startTs = System.currentTimeMillis(); + int totalElements = jdbcTemplate.queryForObject(CF_COUNT_QUERY, Collections.emptyMap(), Integer.class); + log.debug("Count query took {} ms", System.currentTimeMillis() - startTs); + startTs = System.currentTimeMillis(); + List> rows = jdbcTemplate.queryForList(String.format(CF_QUERY, pageable.getPageSize(), pageable.getOffset()), Collections.emptyMap()); + log.debug("Main query took {} ms", System.currentTimeMillis() - startTs); + int totalPages = pageable.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageable.getPageSize()) : 1; + boolean hasNext = pageable.getPageSize() > 0 && totalElements > pageable.getOffset() + rows.size(); + var data = rows.stream().map(row -> { + + UUID id = (UUID) row.get("id"); + long createdTime = (long) row.get("created_time"); + UUID tenantId = (UUID) row.get("tenant_id"); + EntityType entityType = EntityType.valueOf((String) row.get("entity_type")); + UUID entityId = (UUID) row.get("entity_id"); + CalculatedFieldType type = CalculatedFieldType.valueOf((String) row.get("type")); + String name = (String) row.get("name"); + int configurationVersion = (int) row.get("configuration_version"); + JsonNode configuration = JacksonUtil.toJsonNode((String) row.get("configuration")); + long version = row.get("version") != null ? (long) row.get("version") : 0; + String debugSettings = (String) row.get("debug_settings"); + Object externalIdObj = row.get("external_id"); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setId(new CalculatedFieldId(id)); + calculatedField.setCreatedTime(createdTime); + calculatedField.setTenantId(TenantId.fromUUID(tenantId)); + calculatedField.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedField.setType(type); + calculatedField.setName(name); + calculatedField.setConfigurationVersion(configurationVersion); + calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); + calculatedField.setVersion(version); + calculatedField.setDebugSettings(JacksonUtil.fromString(debugSettings, DebugSettings.class)); + calculatedField.setExternalId(externalIdObj != null ? new CalculatedFieldId(UUID.fromString((String) externalIdObj)) : null); + + return calculatedField; + }).collect(Collectors.toList()); + return new PageData<>(data, totalPages, totalElements, hasNext); + }); + } + + @Override + public PageData findCalculatedFieldLinks(Pageable pageable) { + return transactionTemplate.execute(status -> { + long startTs = System.currentTimeMillis(); + int totalElements = jdbcTemplate.queryForObject(CFL_COUNT_QUERY, Collections.emptyMap(), Integer.class); + log.debug("Count query took {} ms", System.currentTimeMillis() - startTs); + startTs = System.currentTimeMillis(); + List> rows = jdbcTemplate.queryForList(String.format(CFL_QUERY, pageable.getPageSize(), pageable.getOffset()), Collections.emptyMap()); + log.debug("Main query took {} ms", System.currentTimeMillis() - startTs); + int totalPages = pageable.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageable.getPageSize()) : 1; + boolean hasNext = pageable.getPageSize() > 0 && totalElements > pageable.getOffset() + rows.size(); + var data = rows.stream().map(row -> { + + UUID id = (UUID) row.get("id"); + long createdTime = (long) row.get("created_time"); + UUID tenantId = (UUID) row.get("tenant_id"); + EntityType entityType = EntityType.valueOf((String) row.get("entity_type")); + UUID entityId = (UUID) row.get("entity_id"); + UUID calculatedFieldId = (UUID) row.get("calculated_field_id"); + JsonNode configuration = JacksonUtil.toJsonNode((String) row.get("configuration")); + + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); + calculatedFieldLink.setId(new CalculatedFieldLinkId(id)); + calculatedFieldLink.setCreatedTime(createdTime); + calculatedFieldLink.setTenantId(new TenantId(tenantId)); + calculatedFieldLink.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedFieldLink.setCalculatedFieldId(new CalculatedFieldId(calculatedFieldId)); + + return calculatedFieldLink; + }).collect(Collectors.toList()); + return new PageData<>(data, totalPages, totalElements, hasNext); + }); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java new file mode 100644 index 0000000000..8922eaca4e --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -0,0 +1,106 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.cf; + +import jakarta.transaction.Transactional; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.cf.CalculatedFieldDao; +import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; +import org.thingsboard.server.dao.sql.JpaAbstractDao; +import org.thingsboard.server.dao.util.SqlDao; + +import java.util.List; +import java.util.UUID; + +@Slf4j +@Component +@AllArgsConstructor +@SqlDao +public class JpaCalculatedFieldDao extends JpaAbstractDao implements CalculatedFieldDao { + + private final CalculatedFieldRepository calculatedFieldRepository; + private final NativeCalculatedFieldRepository nativeCalculatedFieldRepository; + + @Override + public List findAllByTenantId(TenantId tenantId) { + return DaoUtil.convertDataList(calculatedFieldRepository.findAllByTenantId(tenantId.getId())); + } + + @Override + public List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId) { + return calculatedFieldRepository.findCalculatedFieldIdsByTenantIdAndEntityId(tenantId.getId(), entityId.getId()); + } + + @Override + public List findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { + return DaoUtil.convertDataList(calculatedFieldRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId())); + } + + @Override + public List findAll() { + return DaoUtil.convertDataList(calculatedFieldRepository.findAll()); + } + + @Override + public PageData findAll(PageLink pageLink) { + log.debug("Try to find calculated fields by pageLink [{}]", pageLink); + return nativeCalculatedFieldRepository.findCalculatedFields(DaoUtil.toPageable(pageLink)); + } + + @Override + public PageData findAllByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink) { + log.debug("Try to find calculated fields by entityId[{}] and pageLink [{}]", entityId, pageLink); + return DaoUtil.toPageData(calculatedFieldRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId(), DaoUtil.toPageable(pageLink))); + } + + @Override + @Transactional + public List removeAllByEntityId(TenantId tenantId, EntityId entityId) { + return DaoUtil.convertDataList(calculatedFieldRepository.removeAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId())); + } + + @Override + public long countCFByEntityId(TenantId tenantId, EntityId entityId) { + return calculatedFieldRepository.countByTenantIdAndEntityId(tenantId.getId(), entityId.getId()); + } + + @Override + protected Class getEntityClass() { + return CalculatedFieldEntity.class; + } + + @Override + protected JpaRepository getRepository() { + return calculatedFieldRepository; + } + + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java new file mode 100644 index 0000000000..2fd98c8bdf --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.cf; + +import com.google.common.util.concurrent.ListenableFuture; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.cf.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; +import org.thingsboard.server.dao.sql.JpaAbstractDao; +import org.thingsboard.server.dao.util.SqlDao; + +import java.util.List; +import java.util.UUID; + +@Slf4j +@Component +@AllArgsConstructor +@SqlDao +public class JpaCalculatedFieldLinkDao extends JpaAbstractDao implements CalculatedFieldLinkDao { + + private final CalculatedFieldLinkRepository calculatedFieldLinkRepository; + private final NativeCalculatedFieldRepository nativeCalculatedFieldRepository; + + @Override + public List findCalculatedFieldLinksByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAllByTenantIdAndCalculatedFieldId(tenantId.getId(), calculatedFieldId.getId())); + } + + @Override + public List findCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { + return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId())); + } + + @Override + public List findAll() { + return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAll()); + } + + @Override + public PageData findAll(PageLink pageLink) { + log.debug("Try to find calculated field links by pageLink [{}]", pageLink); + return nativeCalculatedFieldRepository.findCalculatedFieldLinks(DaoUtil.toPageable(pageLink)); + } + + @Override + protected Class getEntityClass() { + return CalculatedFieldLinkEntity.class; + } + + @Override + protected JpaRepository getRepository() { + return calculatedFieldLinkRepository; + } + + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD_LINK; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java new file mode 100644 index 0000000000..f37a5764a0 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.cf; + +import org.springframework.data.domain.Pageable; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.page.PageData; + +public interface NativeCalculatedFieldRepository { + + PageData findCalculatedFields(Pageable pageable); + + PageData findCalculatedFieldLinks(Pageable pageable); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardInfoRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardInfoRepository.java index cddbe3e347..7624ddc738 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardInfoRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardInfoRepository.java @@ -78,13 +78,21 @@ public interface DashboardInfoRepository extends JpaRepository findByTenantAndImageLink(@Param("tenantId") UUID tenantId, @Param("imageLink") String imageLink, @Param("lmt") int lmt); + List findByTenantAndImageLink(@Param("tenantId") UUID tenantId, @Param("imageLink") String imageLink, @Param("limit") int limit); @Query(nativeQuery = true, - value = "SELECT * FROM dashboard d WHERE d.image = :imageLink or d.configuration ILIKE CONCAT('%\"', :imageLink, '\"%') limit :lmt" + value = "SELECT * FROM dashboard d WHERE d.image = :imageLink or d.configuration ILIKE CONCAT('%\"', :imageLink, '\"%') limit :limit" ) - List findByImageLink(@Param("imageLink") String imageLink, @Param("lmt") int lmt); + List findByImageLink(@Param("imageLink") String imageLink, @Param("limit") int limit); + + @Query(value = "SELECT * FROM dashboard d WHERE d.tenant_id = :tenantId and d.configuration ILIKE CONCAT('%', :link, '%') limit :limit", + nativeQuery = true) + List findDashboardInfosByTenantIdAndResourceLink(@Param("tenantId") UUID tenantId, @Param("link") String link, @Param("limit") int limit); + + @Query(value = "SELECT * FROM dashboard d WHERE d.configuration ILIKE CONCAT('%', :link, '%') limit :limit", + nativeQuery = true) + List findDashboardInfosByResourceLink(@Param("link") String link, @Param("limit") int limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java index 249e9d6e5b..bc07139725 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java @@ -133,4 +133,15 @@ public class JpaDashboardInfoDao extends JpaAbstractDao findByImageLink(String imageLink, int limit) { return DaoUtil.convertDataList(dashboardInfoRepository.findByImageLink(imageLink, limit)); } + + @Override + public List findByTenantIdAndResourceLink(TenantId tenantId, String url, int limit) { + return DaoUtil.convertDataList(dashboardInfoRepository.findDashboardInfosByTenantIdAndResourceLink(tenantId.getId(), url, limit)); + } + + @Override + public List findByResourceLink(String link, int limit) { + return DaoUtil.convertDataList(dashboardInfoRepository.findDashboardInfosByResourceLink(link, limit)); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/AbstractNativeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/AbstractNativeRepository.java new file mode 100644 index 0000000000..bba84503f4 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/AbstractNativeRepository.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.device; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Pageable; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.server.common.data.page.PageData; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +@RequiredArgsConstructor +@Repository +@Slf4j +public class AbstractNativeRepository { + + private final NamedParameterJdbcTemplate jdbcTemplate; + private final TransactionTemplate transactionTemplate; + + protected PageData find(String countQuery, String findQuery, Pageable pageable, Function, T> mapper) { + return transactionTemplate.execute(status -> { + long startTs = System.currentTimeMillis(); + int totalElements = jdbcTemplate.queryForObject(countQuery, Collections.emptyMap(), Integer.class); + log.debug("Count query took {} ms", System.currentTimeMillis() - startTs); + startTs = System.currentTimeMillis(); + List> rows = jdbcTemplate.queryForList(String.format(findQuery, pageable.getPageSize(), pageable.getOffset()), Collections.emptyMap()); + log.debug("Main query took {} ms", System.currentTimeMillis() - startTs); + int totalPages = pageable.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageable.getPageSize()) : 1; + boolean hasNext = pageable.getPageSize() > 0 && totalElements > pageable.getOffset() + rows.size(); + var data = rows.stream().map(mapper).collect(Collectors.toList()); + return new PageData<>(data, totalPages, totalElements, hasNext); + }); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java new file mode 100644 index 0000000000..43d66e2ff0 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.device; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Pageable; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.server.common.data.DeviceIdInfo; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; + +import java.util.UUID; + +@Repository +@Slf4j +public class DefaultNativeAssetRepository extends AbstractNativeRepository implements NativeAssetRepository { + + private final String COUNT_QUERY = "SELECT count(id) FROM asset;"; + + public DefaultNativeAssetRepository(NamedParameterJdbcTemplate jdbcTemplate, TransactionTemplate transactionTemplate) { + super(jdbcTemplate, transactionTemplate); + } + + @Override + public PageData findProfileEntityIdInfos(Pageable pageable) { + String PROFILE_DEVICE_ID_INFO_QUERY = "SELECT tenant_id as tenantId, asset_profile_id as profileId, id as id FROM asset ORDER BY created_time ASC LIMIT %s OFFSET %s"; + return find(COUNT_QUERY, PROFILE_DEVICE_ID_INFO_QUERY, pageable, row -> { + AssetId id = new AssetId((UUID) row.get("id")); + AssetProfileId profileId = new AssetProfileId((UUID) row.get("profileId")); + var tenantIdObj = row.get("tenantId"); + return ProfileEntityIdInfo.create(tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(), profileId, id); + }); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java index 4556ac555e..776dedc2d5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java @@ -15,50 +15,50 @@ */ package org.thingsboard.server.dao.sql.device; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.domain.Pageable; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.stereotype.Repository; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.server.common.data.DeviceIdInfo; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; -import java.util.Collections; -import java.util.List; -import java.util.Map; import java.util.UUID; -import java.util.stream.Collectors; -@RequiredArgsConstructor @Repository @Slf4j -public class DefaultNativeDeviceRepository implements NativeDeviceRepository { +public class DefaultNativeDeviceRepository extends AbstractNativeRepository implements NativeDeviceRepository { private final String COUNT_QUERY = "SELECT count(id) FROM device;"; - private final String QUERY = "SELECT tenant_id as tenantId, customer_id as customerId, id as id FROM device ORDER BY created_time ASC LIMIT %s OFFSET %s"; - private final NamedParameterJdbcTemplate jdbcTemplate; - private final TransactionTemplate transactionTemplate; + + public DefaultNativeDeviceRepository(NamedParameterJdbcTemplate jdbcTemplate, TransactionTemplate transactionTemplate) { + super(jdbcTemplate, transactionTemplate); + } @Override public PageData findDeviceIdInfos(Pageable pageable) { - return transactionTemplate.execute(status -> { - long startTs = System.currentTimeMillis(); - int totalElements = jdbcTemplate.queryForObject(COUNT_QUERY, Collections.emptyMap(), Integer.class); - log.debug("Count query took {} ms", System.currentTimeMillis() - startTs); - startTs = System.currentTimeMillis(); - List> rows = jdbcTemplate.queryForList(String.format(QUERY, pageable.getPageSize(), pageable.getOffset()), Collections.emptyMap()); - log.debug("Main query took {} ms", System.currentTimeMillis() - startTs); - int totalPages = pageable.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageable.getPageSize()) : 1; - boolean hasNext = pageable.getPageSize() > 0 && totalElements > pageable.getOffset() + rows.size(); - var data = rows.stream().map(row -> { - UUID id = (UUID) row.get("id"); - var tenantIdObj = row.get("tenantId"); - var customerIdObj = row.get("customerId"); - return new DeviceIdInfo(tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(), customerIdObj != null ? (UUID) customerIdObj : null, id); - }).collect(Collectors.toList()); - return new PageData<>(data, totalPages, totalElements, hasNext); + String DEVICE_ID_INFO_QUERY = "SELECT tenant_id as tenantId, customer_id as customerId, id as id FROM device ORDER BY created_time ASC LIMIT %s OFFSET %s"; + return find(COUNT_QUERY, DEVICE_ID_INFO_QUERY, pageable, row -> { + UUID id = (UUID) row.get("id"); + var tenantIdObj = row.get("tenantId"); + var customerIdObj = row.get("customerId"); + return new DeviceIdInfo(tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(), customerIdObj != null ? (UUID) customerIdObj : null, id); }); } + + @Override + public PageData findProfileEntityIdInfos(Pageable pageable) { + String PROFILE_DEVICE_ID_INFO_QUERY = "SELECT tenant_id as tenantId, device_profile_id as profileId, id as id FROM device ORDER BY created_time ASC LIMIT %s OFFSET %s"; + return find(COUNT_QUERY, PROFILE_DEVICE_ID_INFO_QUERY, pageable, row -> { + DeviceId id = new DeviceId((UUID) row.get("id")); + DeviceProfileId profileId = new DeviceProfileId((UUID) row.get("profileId")); + var tenantIdObj = row.get("tenantId"); + return ProfileEntityIdInfo.create(tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(), profileId, id); + }); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java index dd960ff6b7..f4c2fed9fa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java @@ -83,6 +83,14 @@ public interface DeviceRepository extends JpaRepository, Exp @Param("textSearch") String textSearch, Pageable pageable); + @Query("SELECT d.id FROM DeviceEntity d WHERE d.tenantId = :tenantId " + + "AND d.deviceProfileId = :deviceProfileId " + + "AND (:textSearch IS NULL OR ilike(d.type, CONCAT('%', :textSearch, '%')) = true)") + Page findIdsByTenantIdAndDeviceProfileId(@Param("tenantId") UUID tenantId, + @Param("deviceProfileId") UUID deviceProfileId, + @Param("textSearch") String textSearch, + Pageable pageable); + @Query("SELECT d FROM DeviceEntity d WHERE d.tenantId = :tenantId " + "AND d.deviceProfileId = :deviceProfileId " + "AND d.firmwareId IS NULL") diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index e5df169ca9..34835f52f1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.DeviceInfoFilter; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.edqs.fields.DeviceFields; import org.thingsboard.server.common.data.id.DeviceId; @@ -175,6 +176,17 @@ public class JpaDeviceDao extends JpaAbstractDao implement DaoUtil.toPageable(pageLink))); } + @Override + public PageData findDeviceIdsByTenantIdAndDeviceProfileId(UUID tenantId, UUID deviceProfileId, PageLink pageLink) { + return DaoUtil.pageToPageData( + deviceRepository.findIdsByTenantIdAndDeviceProfileId( + tenantId, + deviceProfileId, + pageLink.getTextSearch(), + DaoUtil.toPageable(pageLink))) + .mapData(DeviceId::new); + } + @Override public PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(UUID tenantId, UUID deviceProfileId, @@ -263,6 +275,12 @@ public class JpaDeviceDao extends JpaAbstractDao implement return nativeDeviceRepository.findDeviceIdInfos(DaoUtil.toPageable(pageLink)); } + @Override + public PageData findProfileEntityIdInfos(PageLink pageLink) { + log.debug("Find profile device id infos by pageLink [{}]", pageLink); + return nativeDeviceRepository.findProfileEntityIdInfos(DaoUtil.toPageable(pageLink)); + } + @Override public Device findByTenantIdAndExternalId(UUID tenantId, UUID externalId) { return DaoUtil.getData(deviceRepository.findByTenantIdAndExternalId(tenantId, externalId)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeAssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeAssetRepository.java new file mode 100644 index 0000000000..e1c1c71708 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeAssetRepository.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.device; + +public interface NativeAssetRepository extends NativeProfileEntityRepository { + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeDeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeDeviceRepository.java index 53f70ed046..c9c783cc46 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeDeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeDeviceRepository.java @@ -17,10 +17,12 @@ package org.thingsboard.server.dao.sql.device; import org.springframework.data.domain.Pageable; import org.thingsboard.server.common.data.DeviceIdInfo; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.page.PageData; -public interface NativeDeviceRepository { +public interface NativeDeviceRepository extends NativeProfileEntityRepository { PageData findDeviceIdInfos(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeProfileEntityRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeProfileEntityRepository.java new file mode 100644 index 0000000000..750f0c8787 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeProfileEntityRepository.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.device; + +import org.springframework.data.domain.Pageable; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; +import org.thingsboard.server.common.data.page.PageData; + +public interface NativeProfileEntityRepository { + + PageData findProfileEntityIdInfos(Pageable pageable); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java new file mode 100644 index 0000000000..482905d362 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java @@ -0,0 +1,145 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.event; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; +import org.thingsboard.server.dao.model.sql.CalculatedFieldDebugEventEntity; + +import java.util.List; +import java.util.UUID; + +public interface CalculatedFieldDebugEventRepository extends EventRepository, JpaRepository { + + @Override + @Query(nativeQuery = true, value = "SELECT * FROM cf_debug_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit") + List findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit); + + @Override + @Query("SELECT e FROM CalculatedFieldDebugEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + Pageable pageable); + + @Query(nativeQuery = true, + value = "SELECT * FROM cf_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:calculatedFieldId IS NULL OR e.cf_id = uuid(:calculatedFieldId)) " + + "AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " + + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + + "AND (:eventArguments IS NULL OR e.e_args ILIKE concat('%', :eventArguments, '%')) " + + "AND (:eventResult IS NULL OR e.e_result ILIKE concat('%', :eventResult, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + , + countQuery = "SELECT count(*) FROM cf_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:calculatedFieldId IS NULL OR e.cf_id = uuid(:calculatedFieldId)) " + + "AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " + + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + + "AND (:eventArguments IS NULL OR e.e_args ILIKE concat('%', :eventArguments, '%')) " + + "AND (:eventResult IS NULL OR e.e_result ILIKE concat('%', :eventResult, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String serviceId, + @Param("calculatedFieldId") UUID calculatedFieldId, + @Param("eventEntityId") String eventEntityId, + @Param("eventEntityType") String eventEntityType, + @Param("msgId") String eventMsgId, + @Param("msgType") String eventMsgType, + @Param("eventArguments") String eventArguments, + @Param("eventResult") String eventResult, + @Param("isError") boolean isError, + @Param("error") String error, + Pageable pageable); + + @Transactional + @Modifying + @Query("DELETE FROM CalculatedFieldDebugEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime); + + @Transactional + @Modifying + @Query(nativeQuery = true, + value = "DELETE FROM cf_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:calculatedFieldId IS NULL OR e.cf_id = uuid(:calculatedFieldId)) " + + "AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " + + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + + "AND (:eventArguments IS NULL OR e.e_args ILIKE concat('%', :eventArguments, '%')) " + + "AND (:eventResult IS NULL OR e.e_result ILIKE concat('%', :eventResult, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))") + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String serviceId, + @Param("calculatedFieldId") UUID calculatedFieldId, + @Param("eventEntityId") String eventEntityId, + @Param("eventEntityType") String eventEntityType, + @Param("msgId") String eventMsgId, + @Param("msgType") String eventMsgType, + @Param("eventArguments") String eventArguments, + @Param("eventResult") String eventResult, + @Param("isError") boolean isError, + @Param("error") String error); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/DedicatedJpaEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/DedicatedJpaEventDao.java index abe95fb8f4..e13dfacf81 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/DedicatedJpaEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/DedicatedJpaEventDao.java @@ -36,10 +36,11 @@ public class DedicatedJpaEventDao extends JpaBaseEventDao { RuleNodeDebugEventRepository ruleNodeDebugEventRepository, RuleChainDebugEventRepository ruleChainDebugEventRepository, ScheduledLogExecutorComponent logExecutor, - StatsFactory statsFactory) { + StatsFactory statsFactory, + CalculatedFieldDebugEventRepository cfDebugEventRepository) { super(partitionConfiguration, partitioningRepository, lcEventRepository, statsEventRepository, errorEventRepository, eventInsertRepository, ruleNodeDebugEventRepository, - ruleChainDebugEventRepository, logExecutor, statsFactory); + ruleChainDebugEventRepository, logExecutor, statsFactory, cfDebugEventRepository); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java index cd25577214..142924eac6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java @@ -25,6 +25,7 @@ import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.support.TransactionCallbackWithoutResult; import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventType; @@ -81,6 +82,9 @@ public class EventInsertRepository { insertStmtMap.put(EventType.DEBUG_RULE_CHAIN, "INSERT INTO " + EventType.DEBUG_RULE_CHAIN.getTable() + " (id, tenant_id, ts, entity_id, service_id, e_message, e_error) " + "VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); + insertStmtMap.put(EventType.DEBUG_CALCULATED_FIELD, "INSERT INTO " + EventType.DEBUG_CALCULATED_FIELD.getTable() + + " (id, tenant_id, ts, entity_id, service_id, cf_id, e_entity_id, e_entity_type, e_msg_id, e_msg_type, e_args, e_result, e_error) " + + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); } public void save(List entities) { @@ -107,6 +111,8 @@ public class EventInsertRepository { return getRuleNodeEventSetter(events); case DEBUG_RULE_CHAIN: return getRuleChainEventSetter(events); + case DEBUG_CALCULATED_FIELD: + return getCalculatedFieldEventSetter(events); default: throw new RuntimeException(eventType + " support is not implemented!"); } @@ -206,6 +212,29 @@ public class EventInsertRepository { }; } + private BatchPreparedStatementSetter getCalculatedFieldEventSetter(List events) { + return new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + CalculatedFieldDebugEvent event = (CalculatedFieldDebugEvent) events.get(i); + setCommonEventFields(ps, event); + safePutUUID(ps, 6, event.getCalculatedFieldId().getId()); + safePutUUID(ps, 7, event.getEventEntity() != null ? event.getEventEntity().getId() : null); + safePutString(ps, 8, event.getEventEntity() != null ? event.getEventEntity().getEntityType().name() : null); + safePutUUID(ps, 9, event.getMsgId()); + safePutString(ps, 10, event.getMsgType()); + safePutString(ps, 11, event.getArguments()); + safePutString(ps, 12, event.getResult()); + safePutString(ps, 13, event.getError()); + } + + @Override + public int getBatchSize() { + return events.size(); + } + }; + } + void safePutString(PreparedStatement ps, int parameterIdx, String value) throws SQLException { if (value != null) { ps.setString(parameterIdx, replaceNullChars(value)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java index 3945c80c43..592ed21f9a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java @@ -24,6 +24,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEventFilter; import org.thingsboard.server.common.data.event.ErrorEventFilter; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; @@ -72,6 +73,7 @@ public class JpaBaseEventDao implements EventDao { private final RuleChainDebugEventRepository ruleChainDebugEventRepository; private final ScheduledLogExecutorComponent logExecutor; private final StatsFactory statsFactory; + private final CalculatedFieldDebugEventRepository calculatedFieldDebugEventRepository; @Value("${sql.events.batch_size:10000}") private int batchSize; @@ -110,6 +112,7 @@ public class JpaBaseEventDao implements EventDao { repositories.put(EventType.ERROR, errorEventRepository); repositories.put(EventType.DEBUG_RULE_NODE, ruleNodeDebugEventRepository); repositories.put(EventType.DEBUG_RULE_CHAIN, ruleChainDebugEventRepository); + repositories.put(EventType.DEBUG_CALCULATED_FIELD, calculatedFieldDebugEventRepository); } @PreDestroy @@ -158,6 +161,8 @@ public class JpaBaseEventDao implements EventDao { return findEventByFilter(tenantId, entityId, (ErrorEventFilter) eventFilter, pageLink); case STATS: return findEventByFilter(tenantId, entityId, (StatisticsEventFilter) eventFilter, pageLink); + case DEBUG_CALCULATED_FIELD: + return findEventByFilter(tenantId, entityId, (CalculatedFieldDebugEventFilter) eventFilter, pageLink); default: throw new RuntimeException("Not supported event type: " + eventFilter.getEventType()); } @@ -193,6 +198,8 @@ public class JpaBaseEventDao implements EventDao { case STATS: removeEventsByFilter(tenantId, entityId, (StatisticsEventFilter) eventFilter, startTime, endTime); break; + case DEBUG_CALCULATED_FIELD: + removeEventsByFilter(tenantId, entityId, (CalculatedFieldDebugEventFilter) eventFilter, startTime, endTime); default: throw new RuntimeException("Not supported event type: " + eventFilter.getEventType()); } @@ -286,6 +293,28 @@ public class JpaBaseEventDao implements EventDao { ); } + private PageData findEventByFilter(UUID tenantId, UUID entityId, CalculatedFieldDebugEventFilter eventFilter, TimePageLink pageLink) { + parseUUID(eventFilter.getEntityId(), "Entity Id"); + parseUUID(eventFilter.getMsgId(), "Message Id"); + return DaoUtil.toPageData( + calculatedFieldDebugEventRepository.findEvents( + tenantId, + entityId, + pageLink.getStartTime(), + pageLink.getEndTime(), + eventFilter.getServer(), + entityId, + eventFilter.getEntityId(), + eventFilter.getEntityType(), + eventFilter.getMsgId(), + eventFilter.getMsgType(), + eventFilter.getArguments(), + eventFilter.getResult(), + eventFilter.isError(), + eventFilter.getErrorStr(), + DaoUtil.toPageable(pageLink, EventEntity.eventColumnMap))); + } + private void removeEventsByFilter(UUID tenantId, UUID entityId, RuleChainDebugEventFilter eventFilter, Long startTime, Long endTime) { ruleChainDebugEventRepository.removeEvents( tenantId, @@ -360,6 +389,26 @@ public class JpaBaseEventDao implements EventDao { ); } + private void removeEventsByFilter(UUID tenantId, UUID entityId, CalculatedFieldDebugEventFilter eventFilter, Long startTime, Long endTime) { + parseUUID(eventFilter.getEntityId(), "Entity Id"); + parseUUID(eventFilter.getMsgId(), "Message Id"); + calculatedFieldDebugEventRepository.removeEvents( + tenantId, + entityId, + startTime, + endTime, + eventFilter.getServer(), + entityId, + eventFilter.getEntityId(), + eventFilter.getEntityType(), + eventFilter.getMsgId(), + eventFilter.getMsgType(), + eventFilter.getArguments(), + eventFilter.getResult(), + eventFilter.isError(), + eventFilter.getErrorStr()); + } + @Override public List findLatestEvents(UUID tenantId, UUID entityId, EventType eventType, int limit) { return DaoUtil.convertDataList(getEventRepository(eventType).findLatestEvents(tenantId, entityId, limit)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java index b9afe8ea4c..98e62fc110 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java @@ -54,6 +54,7 @@ public interface ApiUsageStateRepository extends JpaRepository :id ORDER BY a.id") + "a.emailExecState, a.smsExecState, a.alarmExecState, a.version) FROM ApiUsageStateEntity a WHERE a.id > :id ORDER BY a.id") List findNextBatch(@Param("id") UUID id, Limit limit); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java index 95c79a2ae4..c728f5d006 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java @@ -193,11 +193,6 @@ public class JpaWidgetTypeDao extends JpaAbstractDao findWidgetTypesNamesByTenantIdAndResourceLink(UUID tenantId, String link) { - return widgetTypeRepository.findNamesByTenantIdAndResourceLink(tenantId, link); - } - @Override public List findWidgetTypeIdsByTenantIdAndFqns(UUID tenantId, List widgetFqns) { var idFqnPairs = widgetTypeRepository.findWidgetTypeIdsByTenantIdAndFqns(tenantId, widgetFqns); @@ -273,6 +268,16 @@ public class JpaWidgetTypeDao extends JpaAbstractDao findByTenantIdAndResourceLink(TenantId tenantId, String link, int limit) { + return DaoUtil.convertDataList(widgetTypeInfoRepository.findWidgetTypeInfosByTenantIdAndResourceLink(tenantId.getId(), link, limit)); + } + + @Override + public List findByResourceLink(String link, int limit) { + return DaoUtil.convertDataList(widgetTypeInfoRepository.findWidgetTypeInfosByResourceLink(link, limit)); + } + @Override public EntityType getEntityType() { return EntityType.WIDGET_TYPE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeInfoRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeInfoRepository.java index 39ba949ca9..dc79280bcf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeInfoRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeInfoRepository.java @@ -204,13 +204,20 @@ public interface WidgetTypeInfoRepository extends JpaRepository findByTenantAndImageUrl(@Param("tenantId") UUID tenantId, @Param("imageLink") String imageLink, @Param("lmt") int lmt); + List findByTenantAndImageUrl(@Param("tenantId") UUID tenantId, @Param("imageLink") String imageLink, @Param("limit") int limit); @Query(nativeQuery = true, value = "SELECT * FROM widget_type_info_view wti WHERE wti.id IN " + - "(select id from widget_type where image = :imageLink or descriptor ILIKE CONCAT('%', :imageLink, '%') limit :lmt)" + "(select id from widget_type where image = :imageLink or descriptor ILIKE CONCAT('%', :imageLink, '%') limit :limit)" ) - List findByImageUrl(@Param("imageLink") String imageLink, @Param("lmt") int lmt); + List findByImageUrl(@Param("imageLink") String imageLink, @Param("limit") int limit); + + @Query(value = "SELECT * FROM widget_type_info_view w WHERE w.tenant_id = :tenantId AND w.descriptor ILIKE CONCAT('%', :link, '%') LIMIT :limit ", nativeQuery = true) + List findWidgetTypeInfosByTenantIdAndResourceLink(@Param("tenantId") UUID tenantId, @Param("link") String link, @Param("limit") int limit); + + @Query(value = "SELECT * FROM widget_type_info_view w WHERE w.descriptor ILIKE CONCAT('%', :link, '%') LIMIT :limit ", nativeQuery = true) + List findWidgetTypeInfosByResourceLink(@Param("link") String link, @Param("limit") int limit); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java index 2d6e1a9b9c..ffa88b4d54 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java @@ -71,12 +71,6 @@ public interface WidgetTypeRepository extends JpaRepository> 'resources' LIKE concat('%', :resourceLink, '%')", - nativeQuery = true) - List findNamesByTenantIdAndResourceLink(@Param("tenantId") UUID tenantId, - @Param("resourceLink") String resourceLink); - @Query("SELECT externalId FROM WidgetTypeDetailsEntity WHERE id = :id") UUID getExternalIdById(@Param("id") UUID id); diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index 0436a5abec..19ce8e8993 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.common.msg.edqs.EdqsService; @@ -161,67 +162,51 @@ public class BaseTimeseriesService implements TimeseriesService { } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { validate(entityId); - List> futures = new ArrayList<>(INSERTS_PER_ENTRY); - saveAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, 0L); - return Futures.transform(Futures.allAsList(futures), SUM_ALL_INTEGERS, MoreExecutors.directExecutor()); + return doSave(tenantId, entityId, List.of(tsKvEntry), 0L, true, true); } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { - return doSave(tenantId, entityId, tsKvEntries, ttl, true); + public ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { + return doSave(tenantId, entityId, tsKvEntries, ttl, true, true); } @Override - public ListenableFuture saveWithoutLatest(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { - return doSave(tenantId, entityId, tsKvEntries, ttl, false); - } - - private ListenableFuture doSave(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl, boolean saveLatest) { - int inserts = saveLatest ? INSERTS_PER_ENTRY : INSERTS_PER_ENTRY_WITHOUT_LATEST; - List> futures = new ArrayList<>(tsKvEntries.size() * inserts); - for (TsKvEntry tsKvEntry : tsKvEntries) { - if (saveLatest) { - saveAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, ttl); - } else { - saveWithoutLatestAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, ttl); - } - } - return Futures.transform(Futures.allAsList(futures), SUM_ALL_INTEGERS, MoreExecutors.directExecutor()); + public ListenableFuture saveWithoutLatest(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { + return doSave(tenantId, entityId, tsKvEntries, ttl, false, true); } @Override - public ListenableFuture> saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntries) { - List> futures = new ArrayList<>(tsKvEntries.size()); - for (TsKvEntry tsKvEntry : tsKvEntries) { - futures.add(doSaveLatest(tenantId, entityId, tsKvEntry)); - } - return Futures.allAsList(futures); - } - - private void saveAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { - doSaveAndRegisterFuturesFor(tenantId, futures, entityId, tsKvEntry, ttl); - futures.add(Futures.transform(doSaveLatest(tenantId, entityId, tsKvEntry), v -> 0, MoreExecutors.directExecutor())); + public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntries) { + return doSave(tenantId, entityId, tsKvEntries, 0L, true, false); } - private ListenableFuture doSaveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { - return Futures.transform(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry), version -> { - edqsService.onUpdate(tenantId, ObjectType.LATEST_TS_KV, new LatestTsKv(entityId, tsKvEntry, version)); - return version; - }, MoreExecutors.directExecutor()); - } - - private void saveWithoutLatestAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { - doSaveAndRegisterFuturesFor(tenantId, futures, entityId, tsKvEntry, ttl); - } - - private void doSaveAndRegisterFuturesFor(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { - if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW)) { + private ListenableFuture doSave(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl, boolean saveLatest, boolean saveTs) { + if (saveTs && entityId.getEntityType().equals(EntityType.ENTITY_VIEW)) { throw new IncorrectParameterException("Telemetry data can't be stored for entity view. Read only"); } - futures.add(timeseriesDao.savePartition(tenantId, entityId, tsKvEntry.getTs(), tsKvEntry.getKey())); - futures.add(timeseriesDao.save(tenantId, entityId, tsKvEntry, ttl)); + List> tsFutures = saveTs ? new ArrayList<>(tsKvEntries.size() * INSERTS_PER_ENTRY_WITHOUT_LATEST) : null; + List> latestFutures = saveLatest ? new ArrayList<>(tsKvEntries.size()) : null; + for (TsKvEntry tsKvEntry : tsKvEntries) { + if (saveTs) { + tsFutures.add(timeseriesDao.savePartition(tenantId, entityId, tsKvEntry.getTs(), tsKvEntry.getKey())); + tsFutures.add(timeseriesDao.save(tenantId, entityId, tsKvEntry, ttl)); + } + if (saveLatest) { + latestFutures.add(Futures.transform(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry), version -> { + edqsService.onUpdate(tenantId, ObjectType.LATEST_TS_KV, new LatestTsKv(entityId, tsKvEntry, version)); + return version; + }, MoreExecutors.directExecutor())); + } + } + ListenableFuture dpsFuture = saveTs ? Futures.transform(Futures.allAsList(tsFutures), SUM_ALL_INTEGERS, MoreExecutors.directExecutor()) : Futures.immediateFuture(0); + ListenableFuture> versionsFuture = saveLatest ? Futures.allAsList(latestFutures) : Futures.immediateFuture(null); + return Futures.whenAllComplete(dpsFuture, versionsFuture).call(() -> { + Integer dataPoints = Futures.getUnchecked(dpsFuture); + List versions = Futures.getUnchecked(versionsFuture); + return TimeseriesSaveResult.of(dataPoints, versions); + }, MoreExecutors.directExecutor()); } private List updateQueriesForEntityView(EntityView entityView, List queries) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java index d4418fff09..1282493650 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -162,6 +162,7 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A log.trace("Executing save [{}]", apiUsageState.getTenantId()); validateId(apiUsageState.getTenantId(), id -> INCORRECT_TENANT_ID + id); validateId(apiUsageState.getId(), "Can't save new usage state. Only update is allowed!"); + apiUsageState.setVersion(null); ApiUsageState savedState = apiUsageStateDao.save(apiUsageState.getTenantId(), apiUsageState); eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedState.getTenantId()).entityId(savedState.getId()) .entity(savedState).build()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeDao.java b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeDao.java index 7302ed41bc..bbb8d026c0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeDao.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.widget.WidgetsBundleWidget; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.ImageContainerDao; +import org.thingsboard.server.dao.ResourceContainerDao; import java.util.List; import java.util.UUID; @@ -35,7 +36,7 @@ import java.util.UUID; /** * The Interface WidgetTypeDao. */ -public interface WidgetTypeDao extends Dao, ExportableEntityDao, ImageContainerDao { +public interface WidgetTypeDao extends Dao, ExportableEntityDao, ImageContainerDao, ResourceContainerDao { /** * Save or update widget type object @@ -97,8 +98,6 @@ public interface WidgetTypeDao extends Dao, ExportableEntityD WidgetTypeDetails findDetailsByTenantIdAndFqn(UUID tenantId, String fqn); - List findWidgetTypesNamesByTenantIdAndResourceLink(UUID tenantId, String link); - List findWidgetTypeIdsByTenantIdAndFqns(UUID tenantId, List widgetFqns); List findWidgetsBundleWidgetsByWidgetsBundleId(UUID tenantId, UUID widgetsBundleId); diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 50460b2b29..5bd34c4f1d 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -14,6 +14,22 @@ -- limitations under the License. -- +-- +-- ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + CREATE TABLE IF NOT EXISTS tb_schema_settings ( schema_version bigint NOT NULL, @@ -702,6 +718,7 @@ CREATE TABLE IF NOT EXISTS api_usage_state ( email_exec varchar(32), sms_exec varchar(32), alarm_exec varchar(32), + version BIGINT DEFAULT 1, CONSTRAINT api_usage_state_unq_key UNIQUE (tenant_id, entity_id) ); @@ -906,3 +923,46 @@ CREATE TABLE IF NOT EXISTS qr_code_settings ( qr_code_config VARCHAR(100000), CONSTRAINT qr_code_settings_tenant_id_unq_key UNIQUE (tenant_id) ); + +CREATE TABLE IF NOT EXISTS calculated_field ( + id uuid NOT NULL CONSTRAINT calculated_field_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid NOT NULL, + entity_type VARCHAR(32), + entity_id uuid NOT NULL, + type varchar(32) NOT NULL, + name varchar(255) NOT NULL, + configuration_version int DEFAULT 0, + configuration varchar(1000000), + version BIGINT DEFAULT 1, + debug_settings varchar(1024), + external_id UUID, + CONSTRAINT calculated_field_unq_key UNIQUE (entity_id, name), + CONSTRAINT calculated_field_external_id_unq_key UNIQUE (tenant_id, external_id) +); + +CREATE TABLE IF NOT EXISTS calculated_field_link ( + id uuid NOT NULL CONSTRAINT calculated_field_link_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid NOT NULL, + entity_type VARCHAR(32), + entity_id uuid NOT NULL, + calculated_field_id uuid NOT NULL, + CONSTRAINT fk_calculated_field_id FOREIGN KEY (calculated_field_id) REFERENCES calculated_field(id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS cf_debug_event ( + id uuid NOT NULL, + tenant_id uuid NOT NULL , + ts bigint NOT NULL, + entity_id uuid NOT NULL, -- calculated field id + service_id varchar, + cf_id uuid NOT NULL, + e_entity_id uuid, -- target entity id + e_entity_type varchar, + e_msg_id uuid, + e_msg_type varchar, + e_args varchar, + e_result varchar, + e_error varchar +) PARTITION BY RANGE (ts); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/ApiUsageStateServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/ApiUsageStateServiceTest.java index 1ffca98a14..d87018573c 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/ApiUsageStateServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/ApiUsageStateServiceTest.java @@ -20,6 +20,7 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.ApiUsageStateValue; +import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; @@ -30,22 +31,42 @@ public class ApiUsageStateServiceTest extends AbstractServiceTest { ApiUsageStateService apiUsageStateService; @Test - public void testFindApiUsageStateByTenantId() { - ApiUsageState apiUsageState = apiUsageStateService.findTenantApiUsageState(tenantId); - Assert.assertNotNull(apiUsageState); + public void testFindTenantApiUsageState() { + ApiUsageState state = apiUsageStateService.findTenantApiUsageState(tenantId); + Assert.assertNotNull(state); } @Test - public void testUpdateApiUsageState(){ - ApiUsageState apiUsageState = apiUsageStateService.findTenantApiUsageState(tenantId); - Assert.assertNotNull(apiUsageState); - Assert.assertTrue(apiUsageState.isTransportEnabled()); - apiUsageState.setTransportState(ApiUsageStateValue.DISABLED); - apiUsageState = apiUsageStateService.update(apiUsageState); - Assert.assertNotNull(apiUsageState); - apiUsageState = apiUsageStateService.findTenantApiUsageState(tenantId); - Assert.assertNotNull(apiUsageState); - Assert.assertFalse(apiUsageState.isTransportEnabled()); + public void testUpdate() { + ApiUsageState state = apiUsageStateService.findTenantApiUsageState(tenantId); + + state.setTransportState(ApiUsageStateValue.DISABLED); + ApiUsageState updated = apiUsageStateService.update(state); + Assert.assertEquals(ApiUsageStateValue.DISABLED, updated.getTransportState()); + } + + @Test + public void testUpdateWithNullId() { + ApiUsageState newState = new ApiUsageState(); + newState.setTenantId(tenantId); + newState.setTransportState(ApiUsageStateValue.ENABLED); + Assert.assertThrows(IncorrectParameterException.class, () -> apiUsageStateService.update(newState)); + } + + @Test + public void testFindApiUsageStateByEntityId() { + ApiUsageState state = apiUsageStateService.findApiUsageStateByEntityId(tenantId); + Assert.assertNotNull(state); + } + + @Test + public void testDeleteByTenantId() { + ApiUsageState state = apiUsageStateService.findTenantApiUsageState(tenantId); + Assert.assertNotNull(state); + + apiUsageStateService.deleteByTenantId(tenantId); + state = apiUsageStateService.findTenantApiUsageState(tenantId); + Assert.assertNull(state); } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index bcbc0f9be1..462e7a894c 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -30,6 +30,14 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -39,6 +47,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.relation.RelationService; @@ -46,7 +55,9 @@ import org.thingsboard.server.dao.relation.RelationService; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @DaoSqlTest @@ -63,6 +74,8 @@ public class AssetServiceTest extends AbstractServiceTest { @Autowired private AssetProfileService assetProfileService; @Autowired + private CalculatedFieldService calculatedFieldService; + @Autowired private PlatformTransactionManager platformTransactionManager; private IdComparator idComparator = new IdComparator<>(); @@ -214,24 +227,24 @@ public class AssetServiceTest extends AbstractServiceTest { public void testFindAssetTypesByTenantId() throws Exception { List assets = new ArrayList<>(); try { - for (int i=0;i<3;i++) { + for (int i = 0; i < 3; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("My asset B"+i); + asset.setName("My asset B" + i); asset.setType("typeB"); assets.add(assetService.saveAsset(asset)); } - for (int i=0;i<7;i++) { + for (int i = 0; i < 7; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("My asset C"+i); + asset.setName("My asset C" + i); asset.setType("typeC"); assets.add(assetService.saveAsset(asset)); } - for (int i=0;i<9;i++) { + for (int i = 0; i < 9; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("My asset A"+i); + asset.setName("My asset A" + i); asset.setType("typeA"); assets.add(assetService.saveAsset(asset)); } @@ -242,7 +255,9 @@ public class AssetServiceTest extends AbstractServiceTest { Assert.assertEquals("typeB", assetTypes.get(1).getType()); Assert.assertEquals("typeC", assetTypes.get(2).getType()); } finally { - assets.forEach((asset) -> { assetService.deleteAsset(tenantId, asset.getId()); }); + assets.forEach((asset) -> { + assetService.deleteAsset(tenantId, asset.getId()); + }); } } @@ -267,10 +282,10 @@ public class AssetServiceTest extends AbstractServiceTest { @Test public void testFindAssetsByTenantId() { List assets = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("Asset"+i); + asset.setName("Asset" + i); asset.setType("default"); assets.add(assetService.saveAsset(asset)); } @@ -303,11 +318,11 @@ public class AssetServiceTest extends AbstractServiceTest { public void testFindAssetsByTenantIdAndName() { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -315,11 +330,11 @@ public class AssetServiceTest extends AbstractServiceTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i=0;i<17;i++) { + for (int i = 0; i < 17; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -381,11 +396,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title1 = "Asset title 1"; String type1 = "typeA"; List assetsType1 = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -394,11 +409,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title2 = "Asset title 2"; String type2 = "typeB"; List assetsType2 = new ArrayList<>(); - for (int i=0;i<17;i++) { + for (int i = 0; i < 17; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); @@ -464,10 +479,10 @@ public class AssetServiceTest extends AbstractServiceTest { CustomerId customerId = customer.getId(); List assets = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("Asset"+i); + asset.setName("Asset" + i); asset.setType("default"); asset = assetService.saveAsset(asset); assets.add(new AssetInfo(assetService.assignAssetToCustomer(tenantId, asset.getId(), customerId), customer.getTitle(), customer.isPublic(), "default")); @@ -508,11 +523,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i=0;i<17;i++) { + for (int i = 0; i < 17; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -521,11 +536,11 @@ public class AssetServiceTest extends AbstractServiceTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -596,11 +611,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title1 = "Asset title 1"; String type1 = "typeC"; List assetsType1 = new ArrayList<>(); - for (int i=0;i<17;i++) { + for (int i = 0; i < 17; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -610,11 +625,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title2 = "Asset title 2"; String type2 = "typeD"; List assetsType2 = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); @@ -848,4 +863,52 @@ public class AssetServiceTest extends AbstractServiceTest { ); } + @Test + public void testDeleteAssetIfReferencedInCalculatedField() { + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName("My asset"); + asset.setType("default"); + Asset savedAsset = assetService.saveAsset(asset); + + Asset assetWithCf = new Asset(); + assetWithCf.setTenantId(tenantId); + assetWithCf.setName("Asset with CF"); + assetWithCf.setType("default"); + Asset savedAssetWithCf = assetService.saveAsset(assetWithCf); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setName("Test CF"); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setEntityId(savedAssetWithCf.getId()); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + argument.setRefEntityId(savedAsset.getId()); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + + config.setArguments(Map.of("T", argument)); + + config.setExpression("T - (100 - H) / 5"); + + Output output = new Output(); + output.setName("output"); + output.setType(OutputType.TIME_SERIES); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> assetService.deleteAsset(tenantId, savedAsset.getId())) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't delete asset that has entity views or is referenced in calculated fields!"); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java new file mode 100644 index 0000000000..5a783355d7 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -0,0 +1,182 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service; + +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.exception.DataValidationException; + +import java.util.Map; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@DaoSqlTest +public class CalculatedFieldServiceTest extends AbstractServiceTest { + + @Autowired + private CalculatedFieldService calculatedFieldService; + @Autowired + private DeviceService deviceService; + + private ListeningExecutorService executor; + + @Before + public void before() { + executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(8, getClass())); + } + + @After + public void after() { + executor.shutdownNow(); + } + + @Test + public void testSaveCalculatedField() { + Device device = createTestDevice(); + CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + + assertThat(savedCalculatedField).isNotNull(); + assertThat(savedCalculatedField.getId()).isNotNull(); + assertThat(savedCalculatedField.getCreatedTime()).isGreaterThan(0); + assertThat(savedCalculatedField.getTenantId()).isEqualTo(calculatedField.getTenantId()); + assertThat(savedCalculatedField.getEntityId()).isEqualTo(calculatedField.getEntityId()); + assertThat(savedCalculatedField.getType()).isEqualTo(calculatedField.getType()); + assertThat(savedCalculatedField.getName()).isEqualTo(calculatedField.getName()); + assertThat(savedCalculatedField.getConfiguration()).isEqualTo(calculatedField.getConfiguration()); + assertThat(savedCalculatedField.getVersion()).isEqualTo(1L); + + savedCalculatedField.setName("Test CF"); + + CalculatedField updatedCalculatedField = calculatedFieldService.save(savedCalculatedField); + + assertThat(updatedCalculatedField.getName()).isEqualTo(savedCalculatedField.getName()); + assertThat(updatedCalculatedField.getVersion()).isEqualTo(savedCalculatedField.getVersion() + 1); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + } + + @Test + public void testSaveCalculatedFieldWithExistingName() { + Device device = createTestDevice(); + CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); + calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Calculated Field with such name is already in exists!"); + } + + @Test + public void testSaveCalculatedFieldWithExistingExternalId() { + Device device = createTestDevice(); + CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); + calculatedField.setExternalId(new CalculatedFieldId(UUID.fromString("2ef69d0a-89cf-4868-86f8-c50551d87ebe"))); + + calculatedFieldService.save(calculatedField); + + calculatedField.setName("Test 2"); + assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Calculated Field with such external id already exists!"); + } + + @Test + public void testFindCalculatedFieldById() { + CalculatedField savedCalculatedField = saveValidCalculatedField(); + CalculatedField fetchedCalculatedField = calculatedFieldService.findById(tenantId, savedCalculatedField.getId()); + + assertThat(fetchedCalculatedField).isEqualTo(savedCalculatedField); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + } + + @Test + public void testDeleteCalculatedField() { + CalculatedField savedCalculatedField = saveValidCalculatedField(); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + + assertThat(calculatedFieldService.findById(tenantId, savedCalculatedField.getId())).isNull(); + } + + private CalculatedField saveValidCalculatedField() { + Device device = createTestDevice(); + CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); + return calculatedFieldService.save(calculatedField); + } + + private CalculatedField getCalculatedField(EntityId entityId, EntityId referencedEntityId) { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setEntityId(entityId); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); + return calculatedField; + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + argument.setRefEntityId(referencedEntityId); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + + config.setArguments(Map.of("T", argument)); + + config.setExpression("T - (100 - H) / 5"); + + Output output = new Output(); + output.setName("output"); + output.setType(OutputType.TIME_SERIES); + + config.setOutput(output); + + return config; + } + + private Device createTestDevice() { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Test"); + return deviceService.saveDevice(device); + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 3e1dbe08a8..daa10e72e9 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -30,14 +30,26 @@ import org.testcontainers.shaded.org.awaitility.Awaitility; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.exception.DataValidationException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -45,13 +57,17 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; +import static org.assertj.core.api.Assertions.assertThatThrownBy; @DaoSqlTest public class CustomerServiceTest extends AbstractServiceTest { @Autowired CustomerService customerService; + @Autowired + CalculatedFieldService calculatedFieldService; + @Autowired + AssetService assetService; static final int TIMEOUT = 30; @@ -343,4 +359,51 @@ public class CustomerServiceTest extends AbstractServiceTest { } } + @Test + public void testDeleteCustomerIfReferencedInCalculatedField() { + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle("My customer"); + Customer savedCustomer = customerService.saveCustomer(customer); + + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName("My asset"); + asset.setType("default"); + Asset savedAsset = assetService.saveAsset(asset); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setName("Test CF"); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setEntityId(savedAsset.getId()); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + argument.setRefEntityId(savedCustomer.getId()); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + + config.setArguments(Map.of("T", argument)); + + config.setExpression("T - (100 - H) / 5"); + + Output output = new Output(); + output.setName("output"); + output.setType(OutputType.TIME_SERIES); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> customerService.deleteCustomer(tenantId, savedCustomer.getId())) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't delete customer that is referenced in calculated fields!"); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index bf9c94dc89..bbbd48aa49 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -39,6 +39,14 @@ import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -50,6 +58,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceProfileService; @@ -64,6 +73,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; @@ -87,6 +97,8 @@ public class DeviceServiceTest extends AbstractServiceTest { @Autowired TenantProfileService tenantProfileService; @Autowired + private CalculatedFieldService calculatedFieldService; + @Autowired private PlatformTransactionManager platformTransactionManager; @SpyBean private DeviceCredentialsDataValidator validator; @@ -1198,4 +1210,43 @@ public class DeviceServiceTest extends AbstractServiceTest { ); } + @Test + public void testDeleteDeviceIfReferencedInCalculatedField() { + Device device = saveDevice(tenantId, "Test Device"); + Device deviceWithCf = saveDevice(tenantId, "Device with CF"); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setName("Test CF"); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setEntityId(deviceWithCf.getId()); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + argument.setRefEntityId(device.getId()); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + + config.setArguments(Map.of("T", argument)); + + config.setExpression("T - (100 - H) / 5"); + + Output output = new Output(); + output.setName("output"); + output.setType(OutputType.TIME_SERIES); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> deviceService.deleteDevice(tenantId, device.getId())) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't delete device that has entity views or is referenced in calculated fields!"); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java index 0e7797ce56..9503f4e23f 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java @@ -20,6 +20,7 @@ import org.junit.Assert; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.entity.EntityDaoService; import org.thingsboard.server.dao.entity.EntityServiceRegistry; import org.thingsboard.server.dao.rule.RuleChainService; @@ -44,4 +45,8 @@ public class EntityServiceRegistryTest extends AbstractServiceTest { Assert.assertTrue(entityServiceRegistry.getServiceByEntityType(EntityType.RULE_NODE) instanceof RuleChainService); } + @Test + public void givenCalculatedFieldLinkEntityType_whenGetServiceByEntityTypeCalled_thenReturnedCalculatedFieldService() { + Assert.assertTrue(entityServiceRegistry.getServiceByEntityType(EntityType.CALCULATED_FIELD_LINK) instanceof CalculatedFieldService); + } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java new file mode 100644 index 0000000000..43fb44431e --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.validator; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.cf.CalculatedFieldDao; +import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.usagerecord.DefaultApiLimitService; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.BDDMockito.given; + +@SpringBootTest(classes = CalculatedFieldDataValidator.class) +public class CalculatedFieldDataValidatorTest { + + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("7b5229e9-166e-41a9-a257-3b1dafad1b04")); + private final CalculatedFieldId CALCULATED_FIELD_ID = new CalculatedFieldId(UUID.fromString("060fbe45-fbb2-4549-abf3-f72a6be3cb9f")); + + @MockBean + private CalculatedFieldDao calculatedFieldDao; + @MockBean + private DefaultApiLimitService apiLimitService; + @SpyBean + private CalculatedFieldDataValidator validator; + + @Test + public void testUpdateNonExistingCalculatedField() { + CalculatedField calculatedField = new CalculatedField(CALCULATED_FIELD_ID); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("Test"); + + given(calculatedFieldDao.findById(TENANT_ID, CALCULATED_FIELD_ID.getId())).willReturn(null); + + assertThatThrownBy(() -> validator.validateUpdate(TENANT_ID, calculatedField)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't update non existing calculated field!"); + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java new file mode 100644 index 0000000000..c477498602 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.validator; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.cf.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.exception.DataValidationException; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.BDDMockito.given; + +@SpringBootTest(classes = CalculatedFieldLinkDataValidator.class) +public class CalculatedFieldLinkDataValidatorTest { + + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("2ba09d99-6143-43dc-b645-381fc0c43ebe")); + private final CalculatedFieldLinkId CALCULATED_FIELD_LINK_ID = new CalculatedFieldLinkId(UUID.fromString("a5609ef4-cb42-43ce-9b23-e090a4878d1c")); + + @MockBean + private CalculatedFieldLinkDao calculatedFieldLinkDao; + @SpyBean + private CalculatedFieldLinkDataValidator validator; + + @Test + public void testUpdateNonExistingCalculatedField() { + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(CALCULATED_FIELD_LINK_ID); + calculatedFieldLink.setCalculatedFieldId(new CalculatedFieldId(UUID.fromString("136477af-fd07-4498-b9c9-54fe50e82992"))); + + given(calculatedFieldLinkDao.findById(TENANT_ID, CALCULATED_FIELD_LINK_ID.getId())).willReturn(null); + + assertThatThrownBy(() -> validator.validateUpdate(TENANT_ID, calculatedFieldLink)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't update non existing calculated field link!"); + } + +} diff --git a/dao/src/test/resources/application-test.properties b/dao/src/test/resources/application-test.properties index 1e11c77f9a..a1bb335ad0 100644 --- a/dao/src/test/resources/application-test.properties +++ b/dao/src/test/resources/application-test.properties @@ -145,6 +145,8 @@ sql.events.batch_threads=2 actors.system.tenant_dispatcher_pool_size=4 actors.system.device_dispatcher_pool_size=8 actors.system.rule_dispatcher_pool_size=12 +actors.system.cfm_dispatcher_pool_size=2 +actors.system.cfe_dispatcher_pool_size=2 transport.sessions.report_timeout=10000 queue.transport_api.request_poll_interval=5 queue.transport_api.response_poll_interval=5 diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetTypeFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetTypeFilterTest.java index 1420dfac0f..0961a7c12c 100644 --- a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetTypeFilterTest.java +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetTypeFilterTest.java @@ -13,21 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.thingsboard.server.edqs.repo; import org.junit.After; diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 5b57768b79..65def6a964 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -183,6 +183,8 @@ public class ContainerTestSuite { .waitingFor("tb-http-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-mqtt-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-mqtt-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-coap-transport", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-lwm2m-transport", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-vc-executor1", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-js-executor", Wait.forLogMessage(TB_JS_EXECUTOR_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) diff --git a/pom.xml b/pom.xml index 8bff2aaa56..564bc15b2a 100755 --- a/pom.xml +++ b/pom.xml @@ -167,7 +167,7 @@ 2.19.0 9.2.0 1.1.10.5 - 9.4.0 + 9.10.0 diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesDeleteRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesDeleteRequest.java index f1142443ea..2ab6923899 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesDeleteRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesDeleteRequest.java @@ -22,21 +22,27 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.ToString; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; import java.util.List; +import java.util.UUID; @Getter @ToString @AllArgsConstructor(access = AccessLevel.PRIVATE) -public class AttributesDeleteRequest { +public class AttributesDeleteRequest implements CalculatedFieldSystemAwareRequest { private final TenantId tenantId; private final EntityId entityId; private final AttributeScope scope; private final List keys; private final boolean notifyDevice; + private final List previousCalculatedFieldIds; + private final UUID tbMsgId; + private final TbMsgType tbMsgType; private final FutureCallback callback; public static Builder builder() { @@ -50,9 +56,13 @@ public class AttributesDeleteRequest { private AttributeScope scope; private List keys; private boolean notifyDevice; + private List previousCalculatedFieldIds; + private UUID tbMsgId; + private TbMsgType tbMsgType; private FutureCallback callback; - Builder() {} + Builder() { + } public Builder tenantId(TenantId tenantId) { this.tenantId = tenantId; @@ -89,6 +99,21 @@ public class AttributesDeleteRequest { return this; } + public Builder previousCalculatedFieldIds(List previousCalculatedFieldIds) { + this.previousCalculatedFieldIds = previousCalculatedFieldIds; + return this; + } + + public Builder tbMsgId(UUID tbMsgId) { + this.tbMsgId = tbMsgId; + return this; + } + + public Builder tbMsgType(TbMsgType tbMsgType) { + this.tbMsgType = tbMsgType; + return this; + } + public Builder callback(FutureCallback callback) { this.callback = callback; return this; @@ -109,7 +134,7 @@ public class AttributesDeleteRequest { } public AttributesDeleteRequest build() { - return new AttributesDeleteRequest(tenantId, entityId, scope, keys, notifyDevice, callback); + return new AttributesDeleteRequest(tenantId, entityId, scope, keys, notifyDevice, previousCalculatedFieldIds, tbMsgId, tbMsgType, callback); } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java index c7d09d1525..7c8e9d317e 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java @@ -22,24 +22,30 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.ToString; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; import java.util.List; +import java.util.UUID; @Getter @ToString @AllArgsConstructor(access = AccessLevel.PRIVATE) -public class AttributesSaveRequest { +public class AttributesSaveRequest implements CalculatedFieldSystemAwareRequest { private final TenantId tenantId; private final EntityId entityId; private final AttributeScope scope; private final List entries; private final boolean notifyDevice; + private final List previousCalculatedFieldIds; + private final UUID tbMsgId; + private final TbMsgType tbMsgType; private final FutureCallback callback; public static Builder builder() { @@ -53,6 +59,9 @@ public class AttributesSaveRequest { private AttributeScope scope; private List entries; private boolean notifyDevice = true; + private List previousCalculatedFieldIds; + private UUID tbMsgId; + private TbMsgType tbMsgType; private FutureCallback callback; Builder() {} @@ -100,6 +109,21 @@ public class AttributesSaveRequest { return this; } + public Builder previousCalculatedFieldIds(List previousCalculatedFieldIds) { + this.previousCalculatedFieldIds = previousCalculatedFieldIds; + return this; + } + + public Builder tbMsgId(UUID tbMsgId) { + this.tbMsgId = tbMsgId; + return this; + } + + public Builder tbMsgType(TbMsgType tbMsgType) { + this.tbMsgType = tbMsgType; + return this; + } + public Builder callback(FutureCallback callback) { this.callback = callback; return this; @@ -120,7 +144,7 @@ public class AttributesSaveRequest { } public AttributesSaveRequest build() { - return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, callback); + return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, previousCalculatedFieldIds, tbMsgId, tbMsgType, callback); } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/CalculatedFieldSystemAwareRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/CalculatedFieldSystemAwareRequest.java new file mode 100644 index 0000000000..fa4c414172 --- /dev/null +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/CalculatedFieldSystemAwareRequest.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.api; + +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.msg.TbMsgType; + +import java.util.List; +import java.util.UUID; + +public interface CalculatedFieldSystemAwareRequest { + + List getPreviousCalculatedFieldIds(); + + UUID getTbMsgId(); + + TbMsgType getTbMsgType(); + +} diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index 74a1b5087f..6ba4eae0e4 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -50,6 +50,7 @@ import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceCredentialsService; @@ -143,12 +144,19 @@ public interface TbContext { void tellFailure(TbMsg msg, Throwable th); /** - * Puts new message to queue for processing by the Root Rule Chain + * Puts new message to queue from TbMsg for processing by the Root Rule Chain * * @param msg - message */ void enqueue(TbMsg msg, Runnable onSuccess, Consumer onFailure); + /** + * Puts new message to custom queue for processing + * + * @param msg - message + */ + void enqueue(TbMsg msg, String queueName, Runnable onSuccess, Consumer onFailure); + /** * Sends message to the nested rule chain. * Fails processing of the message if the nested rule chain is not found. @@ -167,13 +175,6 @@ public interface TbContext { */ void output(TbMsg msg, String relationType); - /** - * Puts new message to custom queue for processing - * - * @param msg - message - */ - void enqueue(TbMsg msg, String queueName, Runnable onSuccess, Consumer onFailure); - void enqueueForTellFailure(TbMsg msg, String failureMessage); void enqueueForTellFailure(TbMsg tbMsg, Throwable t); @@ -357,6 +358,8 @@ public interface TbContext { SlackService getSlackService(); + CalculatedFieldService getCalculatedFieldService(); + boolean isExternalNodeForceAck(); /** diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesDeleteRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesDeleteRequest.java index 01cad78b98..2ff431c928 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesDeleteRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesDeleteRequest.java @@ -20,21 +20,27 @@ import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.ToString; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.msg.TbMsgType; import java.util.List; +import java.util.UUID; @Getter @ToString @AllArgsConstructor(access = AccessLevel.PRIVATE) -public class TimeseriesDeleteRequest { +public class TimeseriesDeleteRequest implements CalculatedFieldSystemAwareRequest { private final TenantId tenantId; private final EntityId entityId; private final List keys; private final List deleteHistoryQueries; + private final List previousCalculatedFieldIds; + private final UUID tbMsgId; + private final TbMsgType tbMsgType; private final FutureCallback> callback; public static Builder builder() { @@ -47,9 +53,13 @@ public class TimeseriesDeleteRequest { private EntityId entityId; private List keys; private List deleteHistoryQueries; + private List previousCalculatedFieldIds; + private UUID tbMsgId; + private TbMsgType tbMsgType; private FutureCallback> callback; - Builder() {} + Builder() { + } public Builder tenantId(TenantId tenantId) { this.tenantId = tenantId; @@ -71,13 +81,28 @@ public class TimeseriesDeleteRequest { return this; } + public Builder previousCalculatedFieldIds(List previousCalculatedFieldIds) { + this.previousCalculatedFieldIds = previousCalculatedFieldIds; + return this; + } + + public Builder tbMsgId(UUID tbMsgId) { + this.tbMsgId = tbMsgId; + return this; + } + + public Builder tbMsgType(TbMsgType tbMsgType) { + this.tbMsgType = tbMsgType; + return this; + } + public Builder callback(FutureCallback> callback) { this.callback = callback; return this; } public TimeseriesDeleteRequest build() { - return new TimeseriesDeleteRequest(tenantId, entityId, keys, deleteHistoryQueries, callback); + return new TimeseriesDeleteRequest(tenantId, entityId, keys, deleteHistoryQueries, previousCalculatedFieldIds, tbMsgId, tbMsgType, callback); } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java index 103b354be9..01af278efb 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java @@ -20,18 +20,24 @@ import com.google.common.util.concurrent.SettableFuture; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; +import org.thingsboard.common.util.NoOpFutureCallback; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; import java.util.List; +import java.util.UUID; + +import static java.util.Objects.requireNonNullElse; @Getter @AllArgsConstructor(access = AccessLevel.PRIVATE) -public class TimeseriesSaveRequest { +public class TimeseriesSaveRequest implements CalculatedFieldSystemAwareRequest { private final TenantId tenantId; private final CustomerId customerId; @@ -39,14 +45,17 @@ public class TimeseriesSaveRequest { private final List entries; private final long ttl; private final Strategy strategy; + private final List previousCalculatedFieldIds; + private final UUID tbMsgId; + private final TbMsgType tbMsgType; private final FutureCallback callback; - public record Strategy(boolean saveTimeseries, boolean saveLatest, boolean sendWsUpdate) { + public record Strategy(boolean saveTimeseries, boolean saveLatest, boolean sendWsUpdate, boolean processCalculatedFields) { - public static final Strategy SAVE_ALL = new Strategy(true, true, true); - public static final Strategy WS_ONLY = new Strategy(false, false, true); - public static final Strategy LATEST_AND_WS = new Strategy(false, true, true); - public static final Strategy SKIP_ALL = new Strategy(false, false, false); + public static final Strategy PROCESS_ALL = new Strategy(true, true, true, true); + public static final Strategy WS_ONLY = new Strategy(false, false, true, false); + public static final Strategy LATEST_AND_WS = new Strategy(false, true, true, false); + public static final Strategy SKIP_ALL = new Strategy(false, false, false, false); } @@ -61,7 +70,10 @@ public class TimeseriesSaveRequest { private EntityId entityId; private List entries; private long ttl; - private Strategy strategy = Strategy.SAVE_ALL; + private Strategy strategy = Strategy.PROCESS_ALL; + private List previousCalculatedFieldIds; + private UUID tbMsgId; + private TbMsgType tbMsgType; private FutureCallback callback; Builder() {} @@ -104,6 +116,21 @@ public class TimeseriesSaveRequest { return this; } + public Builder previousCalculatedFieldIds(List previousCalculatedFieldIds) { + this.previousCalculatedFieldIds = previousCalculatedFieldIds; + return this; + } + + public Builder tbMsgId(UUID tbMsgId) { + this.tbMsgId = tbMsgId; + return this; + } + + public Builder tbMsgType(TbMsgType tbMsgType) { + this.tbMsgType = tbMsgType; + return this; + } + public Builder callback(FutureCallback callback) { this.callback = callback; return this; @@ -124,7 +151,10 @@ public class TimeseriesSaveRequest { } public TimeseriesSaveRequest build() { - return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, strategy, callback); + return new TimeseriesSaveRequest( + tenantId, customerId, entityId, entries, ttl, strategy, + previousCalculatedFieldIds, tbMsgId, tbMsgType, requireNonNullElse(callback, NoOpFutureCallback.instance()) + ); } } diff --git a/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java b/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java index ecefcca1ea..a67a11ad3d 100644 --- a/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java +++ b/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java @@ -16,36 +16,51 @@ package org.thingsboard.rule.engine.api; import org.junit.jupiter.api.Test; +import org.thingsboard.common.util.NoOpFutureCallback; import static org.assertj.core.api.Assertions.assertThat; class TimeseriesSaveRequestTest { @Test - void testDefaultSaveStrategyIsSaveAll() { + void testDefaultSaveStrategyIsProcessAll() { var request = TimeseriesSaveRequest.builder().build(); - assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL); + assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.PROCESS_ALL); } @Test void testSaveAllStrategy() { - assertThat(TimeseriesSaveRequest.Strategy.SAVE_ALL).isEqualTo(new TimeseriesSaveRequest.Strategy(true, true, true)); + assertThat(TimeseriesSaveRequest.Strategy.PROCESS_ALL).isEqualTo(new TimeseriesSaveRequest.Strategy(true, true, true, true)); } @Test void testWsOnlyStrategy() { - assertThat(TimeseriesSaveRequest.Strategy.WS_ONLY).isEqualTo(new TimeseriesSaveRequest.Strategy(false, false, true)); + assertThat(TimeseriesSaveRequest.Strategy.WS_ONLY).isEqualTo(new TimeseriesSaveRequest.Strategy(false, false, true, false)); } @Test void testLatestAndWsStrategy() { - assertThat(TimeseriesSaveRequest.Strategy.LATEST_AND_WS).isEqualTo(new TimeseriesSaveRequest.Strategy(false, true, true)); + assertThat(TimeseriesSaveRequest.Strategy.LATEST_AND_WS).isEqualTo(new TimeseriesSaveRequest.Strategy(false, true, true, false)); } @Test void testSkipAllStrategy() { - assertThat(TimeseriesSaveRequest.Strategy.SKIP_ALL).isEqualTo(new TimeseriesSaveRequest.Strategy(false, false, false)); + assertThat(TimeseriesSaveRequest.Strategy.SKIP_ALL).isEqualTo(new TimeseriesSaveRequest.Strategy(false, false, false, false)); + } + + @Test + void testDefaultCallbackIsNoOp() { + var request = TimeseriesSaveRequest.builder().build(); + + assertThat(request.getCallback()).isEqualTo(NoOpFutureCallback.instance()); + } + + @Test + void testNullCallbackIsNoOp() { + var request = TimeseriesSaveRequest.builder().callback(null).build(); + + assertThat(request.getCallback()).isEqualTo(NoOpFutureCallback.instance()); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/AbstractTbMsgPushNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/AbstractTbMsgPushNode.java index 77f4937296..d1016ee4fe 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/AbstractTbMsgPushNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/edge/AbstractTbMsgPushNode.java @@ -39,6 +39,8 @@ import java.util.UUID; import static org.thingsboard.server.common.data.msg.TbMsgType.ACTIVITY_EVENT; import static org.thingsboard.server.common.data.msg.TbMsgType.ALARM; +import static org.thingsboard.server.common.data.msg.TbMsgType.ALARM_ACK; +import static org.thingsboard.server.common.data.msg.TbMsgType.ALARM_CLEAR; import static org.thingsboard.server.common.data.msg.TbMsgType.ATTRIBUTES_DELETED; import static org.thingsboard.server.common.data.msg.TbMsgType.ATTRIBUTES_UPDATED; import static org.thingsboard.server.common.data.msg.TbMsgType.CONNECT_EVENT; @@ -79,6 +81,9 @@ public abstract class AbstractTbMsgPushNode metadata = msg.getMetaData().getData(); EdgeEventActionType actionType = getEdgeEventActionTypeByMsgType(msg); @@ -165,6 +170,8 @@ public abstract class AbstractTbMsgPushNodeTime series: save time series data to a ts_kv table in a DB.
  • Latest values: save time series data to a ts_kv_latest table in a DB.
  • WebSockets: notify WebSockets subscriptions about time series data updates.
    • +
  • Calculated fields: notify calculated fields about time series data updates.
    • For each action, three processing strategies are available: @@ -90,7 +91,7 @@ import static org.thingsboard.server.common.data.msg.TbMsgType.POST_TELEMETRY_RE By default, the timestamp is taken from metadata.ts. You can enable Use server timestamp to always use the current server time instead. This is particularly useful in sequential processing scenarios where messages may arrive with out-of-order timestamps from - multiple sources. Note that the DB layer may ignore older records for attributes and latest values, + multiple sources. Note that the DB layer may ignore "outdated" records for attributes and latest values, so enabling Use server timestamp can ensure correct ordering.

    The TTL is taken first from metadata.TTL. If absent, the node configuration’s default @@ -137,7 +138,7 @@ public class TbMsgTimeseriesNode implements TbNode { TimeseriesSaveRequest.Strategy strategy = determineSaveStrategy(ts, msg.getOriginator().getId()); // short-circuit - if (!strategy.saveTimeseries() && !strategy.saveLatest() && !strategy.sendWsUpdate()) { + if (!strategy.saveTimeseries() && !strategy.saveLatest() && !strategy.sendWsUpdate() && !strategy.processCalculatedFields()) { ctx.tellSuccess(msg); return; } @@ -166,6 +167,9 @@ public class TbMsgTimeseriesNode implements TbNode { .entries(tsKvEntryList) .ttl(ttl) .strategy(strategy) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) .callback(new TelemetryNodeCallback(ctx, msg)) .build()); } @@ -176,20 +180,21 @@ public class TbMsgTimeseriesNode implements TbNode { private TimeseriesSaveRequest.Strategy determineSaveStrategy(long ts, UUID originatorUuid) { if (processingSettings instanceof OnEveryMessage) { - return TimeseriesSaveRequest.Strategy.SAVE_ALL; + return TimeseriesSaveRequest.Strategy.PROCESS_ALL; } if (processingSettings instanceof WebSocketsOnly) { return TimeseriesSaveRequest.Strategy.WS_ONLY; } if (processingSettings instanceof Deduplicate deduplicate) { boolean isFirstMsgInInterval = deduplicate.getProcessingStrategy().shouldProcess(ts, originatorUuid); - return isFirstMsgInInterval ? TimeseriesSaveRequest.Strategy.SAVE_ALL : TimeseriesSaveRequest.Strategy.SKIP_ALL; + return isFirstMsgInInterval ? TimeseriesSaveRequest.Strategy.PROCESS_ALL : TimeseriesSaveRequest.Strategy.SKIP_ALL; } if (processingSettings instanceof Advanced advanced) { return new TimeseriesSaveRequest.Strategy( advanced.timeseries().shouldProcess(ts, originatorUuid), advanced.latest().shouldProcess(ts, originatorUuid), - advanced.webSockets().shouldProcess(ts, originatorUuid) + advanced.webSockets().shouldProcess(ts, originatorUuid), + advanced.calculatedFields().shouldProcess(ts, originatorUuid) ); } // should not happen @@ -212,6 +217,7 @@ public class TbMsgTimeseriesNode implements TbNode { var skipLatestProcessingSettings = new Advanced( ProcessingStrategy.onEveryMessage(), ProcessingStrategy.skip(), + ProcessingStrategy.onEveryMessage(), ProcessingStrategy.onEveryMessage() ); ((ObjectNode) oldConfiguration).set("processingSettings", JacksonUtil.valueToTree(skipLatestProcessingSettings)); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java index ce41e08475..a6cd278e67 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java @@ -83,12 +83,18 @@ public class TbMsgTimeseriesNodeConfiguration implements NodeConfiguration { assertThat(request.getEntries()).size().isOne(); - assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL); + assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.PROCESS_ALL); })); TbMsg resultMsg = msgCaptor.getValue(); @@ -569,7 +569,7 @@ public class TbMathNodeTest { verify(ctx, timeout(TIMEOUT)).tellSuccess(msgCaptor.capture()); verify(telemetryService, times(1)).saveTimeseries(assertArg(request -> { assertThat(request.getEntries()).size().isOne(); - assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL); + assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.PROCESS_ALL); })); TbMsg resultMsg = msgCaptor.getValue(); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java index de651e9841..ea8167ca18 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java @@ -208,7 +208,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { assertThat(request.getEntityId()).isEqualTo(DEVICE_ID); assertThat(request.getEntries()).usingRecursiveFieldByFieldElementComparatorIgnoringFields("ts").containsExactlyElementsOf(expectedList); assertThat(request.getTtl()).isEqualTo(extractTtlAsSeconds(tenantProfile)); - assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL); + assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.PROCESS_ALL); assertThat(request.getCallback()).isInstanceOf(TelemetryNodeCallback.class); })); verify(ctxMock).tellSuccess(msg); @@ -223,7 +223,8 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { var timeseriesStrategy = ProcessingStrategy.onEveryMessage(); var latestStrategy = ProcessingStrategy.skip(); var webSockets = ProcessingStrategy.onEveryMessage(); - var processingSettings = new TbMsgTimeseriesNodeConfiguration.ProcessingSettings.Advanced(timeseriesStrategy, latestStrategy, webSockets); + var calculatedFields = ProcessingStrategy.onEveryMessage(); + var processingSettings = new TbMsgTimeseriesNodeConfiguration.ProcessingSettings.Advanced(timeseriesStrategy, latestStrategy, webSockets, calculatedFields); config.setProcessingSettings(processingSettings); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); @@ -265,7 +266,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { assertThat(request.getEntityId()).isEqualTo(DEVICE_ID); assertThat(request.getEntries()).containsExactlyElementsOf(expectedList); assertThat(request.getTtl()).isEqualTo(config.getDefaultTTL()); - assertThat(request.getStrategy()).isEqualTo(new TimeseriesSaveRequest.Strategy(true, false, true)); + assertThat(request.getStrategy()).isEqualTo(new TimeseriesSaveRequest.Strategy(true, false, true, true)); assertThat(request.getCallback()).isInstanceOf(TelemetryNodeCallback.class); })); verify(ctxMock).tellSuccess(msg); @@ -304,7 +305,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { assertThat(request.getCustomerId()).isNull(); assertThat(request.getEntityId()).isEqualTo(DEVICE_ID); assertThat(request.getTtl()).isEqualTo(expectedTtl); - assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL); + assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.PROCESS_ALL); assertThat(request.getCallback()).isInstanceOf(TelemetryNodeCallback.class); })); } @@ -353,7 +354,10 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entityId(msg.getOriginator()) .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) - .strategy(TimeseriesSaveRequest.Strategy.SAVE_ALL) + .strategy(TimeseriesSaveRequest.Strategy.PROCESS_ALL) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) .build(); node.onMsg(ctxMock, msg); @@ -388,7 +392,10 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entityId(msg.getOriginator()) .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) - .strategy(TimeseriesSaveRequest.Strategy.SAVE_ALL) + .strategy(TimeseriesSaveRequest.Strategy.PROCESS_ALL) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) .build(); node.onMsg(ctxMock, msg); @@ -424,6 +431,9 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) .strategy(TimeseriesSaveRequest.Strategy.WS_ONLY) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) .build(); node.onMsg(ctxMock, msg); @@ -441,6 +451,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { public void givenAdvancedProcessingSettingsWithOnEveryMessageStrategiesForAllActionsAndSameMessageTwoTimes_whenOnMsg_thenPersistSameMessageTwoTimes() throws TbNodeException { // GIVEN config.setProcessingSettings(new TbMsgTimeseriesNodeConfiguration.ProcessingSettings.Advanced( + ProcessingStrategy.onEveryMessage(), ProcessingStrategy.onEveryMessage(), ProcessingStrategy.onEveryMessage(), ProcessingStrategy.onEveryMessage() @@ -462,7 +473,11 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entityId(msg.getOriginator()) .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) - .strategy(TimeseriesSaveRequest.Strategy.SAVE_ALL) + .strategy(TimeseriesSaveRequest.Strategy.PROCESS_ALL) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) + .callback(new TelemetryNodeCallback(ctxMock, msg)) .build(); node.onMsg(ctxMock, msg); @@ -482,7 +497,8 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { config.setProcessingSettings(new TbMsgTimeseriesNodeConfiguration.ProcessingSettings.Advanced( ProcessingStrategy.deduplicate(1), ProcessingStrategy.deduplicate(2), - ProcessingStrategy.deduplicate(3) + ProcessingStrategy.deduplicate(3), + ProcessingStrategy.deduplicate(4) )); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); @@ -490,6 +506,8 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { long ts1 = 500L; long ts2 = 1500L; long ts3 = 2500L; + long ts4 = 3500L; + long ts5 = 4500L; // WHEN-THEN node.onMsg(ctxMock, TbMsg.newMsg() @@ -499,7 +517,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts1)))) .build()); then(telemetryServiceMock).should().saveTimeseries(assertArg( - actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL) + actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.PROCESS_ALL) )); clearInvocations(telemetryServiceMock); @@ -511,7 +529,9 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts2)))) .build()); then(telemetryServiceMock).should().saveTimeseries(assertArg( - actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo(new TimeseriesSaveRequest.Strategy(true, false, false)) + actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo( + new TimeseriesSaveRequest.Strategy(true, false, false, false) + ) )); clearInvocations(telemetryServiceMock); @@ -523,7 +543,37 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts3)))) .build()); then(telemetryServiceMock).should().saveTimeseries(assertArg( - actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo(new TimeseriesSaveRequest.Strategy(true, true, false)) + actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo( + new TimeseriesSaveRequest.Strategy(true, true, false, false) + ) + )); + + clearInvocations(telemetryServiceMock); + + node.onMsg(ctxMock, TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(DEVICE_ID) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts4)))) + .build()); + then(telemetryServiceMock).should().saveTimeseries(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo( + new TimeseriesSaveRequest.Strategy(true, false, true, false) + ) + )); + + clearInvocations(telemetryServiceMock); + + node.onMsg(ctxMock, TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(DEVICE_ID) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts5)))) + .build()); + then(telemetryServiceMock).should().saveTimeseries(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo( + new TimeseriesSaveRequest.Strategy(true, true, false, true) + ) )); } @@ -531,6 +581,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { public void givenAdvancedProcessingSettingsWithSkipStrategiesForAllActionsAndSameMessageTwoTimes_whenOnMsg_thenSkipsSameMessageTwoTimes() throws TbNodeException { // GIVEN config.setProcessingSettings(new TbMsgTimeseriesNodeConfiguration.ProcessingSettings.Advanced( + ProcessingStrategy.skip(), ProcessingStrategy.skip(), ProcessingStrategy.skip(), ProcessingStrategy.skip() @@ -631,6 +682,9 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { }, "webSockets": { "type": "ON_EVERY_MESSAGE" + }, + "calculatedFields": { + "type": "ON_EVERY_MESSAGE" } } }""") diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java index 8bcb8503d8..38417c3922 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java @@ -43,6 +43,8 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.domain.Domain; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -67,6 +69,7 @@ import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceService; @@ -155,6 +158,8 @@ public class TenantIdLoaderTest { private MobileAppService mobileAppService; @Mock private MobileAppBundleService mobileAppBundleService; + @Mock + private CalculatedFieldService calculatedFieldService; private TenantId tenantId; private TenantProfileId tenantProfileId; @@ -402,6 +407,18 @@ public class TenantIdLoaderTest { when(ctx.getMobileAppBundleService()).thenReturn(mobileAppBundleService); doReturn(mobileAppBundle).when(mobileAppBundleService).findMobileAppBundleById(eq(tenantId), any()); break; + case CALCULATED_FIELD: + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + when(ctx.getCalculatedFieldService()).thenReturn(calculatedFieldService); + doReturn(calculatedField).when(calculatedFieldService).findById(eq(tenantId), any()); + break; + case CALCULATED_FIELD_LINK: + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); + calculatedFieldLink.setTenantId(tenantId); + when(ctx.getCalculatedFieldService()).thenReturn(calculatedFieldService); + doReturn(calculatedFieldLink).when(calculatedFieldService).findCalculatedFieldLinkById(eq(tenantId), any()); + break; default: throw new RuntimeException("Unexpected originator EntityType " + entityType); } diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 5740f199f8..f1d325b81c 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -133,8 +133,6 @@ redis: transport: # Local CoAP transport parameters coap: - # Enable/disable coap transport protocol. - enabled: "${COAP_ENABLED:true}" # CoaP processing timeout in milliseconds timeout: "${COAP_TIMEOUT:10000}" # CoaP piggyback response timeout in milliseconds @@ -173,8 +171,6 @@ transport: # CoAP server parameters coap: - # Enable/disable coap transport protocol. - enabled: "${COAP_SERVER_ENABLED:true}" # CoAP bind-address bind_address: "${COAP_BIND_ADDRESS:0.0.0.0}" # CoAP bind port diff --git a/ui-ngx/src/app/core/auth/auth.models.ts b/ui-ngx/src/app/core/auth/auth.models.ts index 55af791156..1944c693ac 100644 --- a/ui-ngx/src/app/core/auth/auth.models.ts +++ b/ui-ngx/src/app/core/auth/auth.models.ts @@ -28,6 +28,8 @@ export interface SysParamsState { userSettings: UserSettings; maxResourceSize: number; maxDebugModeDurationMinutes: number; + maxDataPointsPerRollingArg: number; + maxArgumentsPerCF: number; ruleChainDebugPerTenantLimitsConfiguration?: string; } diff --git a/ui-ngx/src/app/core/auth/auth.reducer.ts b/ui-ngx/src/app/core/auth/auth.reducer.ts index a460cf35bb..3ecf70074c 100644 --- a/ui-ngx/src/app/core/auth/auth.reducer.ts +++ b/ui-ngx/src/app/core/auth/auth.reducer.ts @@ -31,6 +31,8 @@ const emptyUserAuthState: AuthPayload = { persistDeviceStateToTelemetry: false, mobileQrEnabled: false, maxResourceSize: 0, + maxArgumentsPerCF: 0, + maxDataPointsPerRollingArg: 0, maxDebugModeDurationMinutes: 0, userSettings: initialUserSettings }; diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts new file mode 100644 index 0000000000..fe5b0f7b52 --- /dev/null +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -0,0 +1,56 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Injectable } from '@angular/core'; +import { defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; +import { Observable } from 'rxjs'; +import { HttpClient } from '@angular/common/http'; +import { PageData } from '@shared/models/page/page-data'; +import { CalculatedField, CalculatedFieldTestScriptInputParams } from '@shared/models/calculated-field.models'; +import { PageLink } from '@shared/models/page/page-link'; +import { EntityId } from '@shared/models/id/entity-id'; +import { EntityTestScriptResult } from '@shared/models/entity.models'; + +@Injectable({ + providedIn: 'root' +}) +export class CalculatedFieldsService { + + constructor( + private http: HttpClient + ) { } + + public getCalculatedFieldById(calculatedFieldId: string, config?: RequestConfig): Observable { + return this.http.get(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); + } + + public saveCalculatedField(calculatedField: CalculatedField, config?: RequestConfig): Observable { + return this.http.post('/api/calculatedField', calculatedField, defaultHttpOptionsFromConfig(config)); + } + + public deleteCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { + return this.http.delete(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); + } + + public getCalculatedFields({ entityType, id }: EntityId, pageLink: PageLink, config?: RequestConfig): Observable> { + return this.http.get>(`/api/${entityType}/${id}/calculatedFields${pageLink.toQuery()}`, + defaultHttpOptionsFromConfig(config)); + } + + public testScript(inputParams: CalculatedFieldTestScriptInputParams, config?: RequestConfig): Observable { + return this.http.post('/api/calculatedField/testScript', inputParams, defaultHttpOptionsFromConfig(config)); + } +} diff --git a/ui-ngx/src/app/core/http/resource.service.ts b/ui-ngx/src/app/core/http/resource.service.ts index 9f0daa73ee..615b721b97 100644 --- a/ui-ngx/src/app/core/http/resource.service.ts +++ b/ui-ngx/src/app/core/http/resource.service.ts @@ -90,8 +90,8 @@ export class ResourceService { return this.http.post('/api/resource', resource, defaultHttpOptionsFromConfig(config)); } - public deleteResource(resourceId: string, config?: RequestConfig) { - return this.http.delete(`/api/resource/${resourceId}`, defaultHttpOptionsFromConfig(config)); + public deleteResource(resourceId: string, force = false, config?: RequestConfig) { + return this.http.delete(`/api/resource/${resourceId}?force=${force}`, defaultHttpOptionsFromConfig(config)); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts new file mode 100644 index 0000000000..6932860be4 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -0,0 +1,293 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { EntityTableColumn, EntityTableConfig } from '@home/models/entity/entities-table-config.models'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { TranslateService } from '@ngx-translate/core'; +import { Direction } from '@shared/models/page/sort-order'; +import { MatDialog } from '@angular/material/dialog'; +import { PageLink } from '@shared/models/page/page-link'; +import { Observable, of } from 'rxjs'; +import { PageData } from '@shared/models/page/page-data'; +import { EntityId } from '@shared/models/id/entity-id'; +import { MINUTE } from '@shared/models/time/time.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { getCurrentAuthState, getCurrentAuthUser } from '@core/auth/auth.selectors'; +import { DestroyRef, Renderer2 } from '@angular/core'; +import { EntityDebugSettings } from '@shared/models/entity.models'; +import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { EntityDebugSettingsPanelComponent } from '@home/components/entity/debug/entity-debug-settings-panel.component'; +import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; +import { catchError, filter, switchMap, tap } from 'rxjs/operators'; +import { + ArgumentType, + CalculatedField, + CalculatedFieldEventArguments, + CalculatedFieldDebugDialogData, + CalculatedFieldDialogData, + CalculatedFieldTestScriptDialogData, + getCalculatedFieldArgumentsEditorCompleter, + getCalculatedFieldArgumentsHighlights, + CalculatedFieldTypeTranslations, +} from '@shared/models/calculated-field.models'; +import { + CalculatedFieldDebugDialogComponent, + CalculatedFieldDialogComponent, + CalculatedFieldScriptTestDialogComponent +} from './components/public-api'; +import { ImportExportService } from '@shared/import-export/import-export.service'; +import { isObject } from '@core/utils'; + +export class CalculatedFieldsTableConfig extends EntityTableConfig { + + // TODO: [Calculated Fields] remove hardcode when BE variable implemented + readonly calculatedFieldsDebugPerTenantLimitsConfiguration = + getCurrentAuthState(this.store)['calculatedFieldsDebugPerTenantLimitsConfiguration'] || '1:1'; + readonly maxDebugModeDuration = getCurrentAuthState(this.store).maxDebugModeDurationMinutes * MINUTE; + readonly tenantId = getCurrentAuthUser(this.store).tenantId; + additionalDebugActionConfig = { + title: this.translate.instant('calculated-fields.see-debug-events'), + action: (calculatedField: CalculatedField) => this.openDebugEventsDialog.call(this, calculatedField), + }; + + constructor(private calculatedFieldsService: CalculatedFieldsService, + private translate: TranslateService, + private dialog: MatDialog, + public entityId: EntityId = null, + private store: Store, + private durationLeft: DurationLeftPipe, + private popoverService: TbPopoverService, + private destroyRef: DestroyRef, + private renderer: Renderer2, + public entityName: string, + private importExportService: ImportExportService + ) { + super(); + this.tableTitle = this.translate.instant('entity.type-calculated-fields'); + this.detailsPanelEnabled = false; + this.pageMode = false; + this.entityType = EntityType.CALCULATED_FIELD; + this.entityTranslations = entityTypeTranslations.get(EntityType.CALCULATED_FIELD); + + this.entitiesFetchFunction = (pageLink: PageLink) => this.fetchCalculatedFields(pageLink); + this.addEntity = this.getCalculatedFieldDialog.bind(this); + this.deleteEntityTitle = (field: CalculatedField) => this.translate.instant('calculated-fields.delete-title', {title: field.name}); + this.deleteEntityContent = () => this.translate.instant('calculated-fields.delete-text'); + this.deleteEntitiesTitle = count => this.translate.instant('calculated-fields.delete-multiple-title', {count}); + this.deleteEntitiesContent = () => this.translate.instant('calculated-fields.delete-multiple-text'); + this.deleteEntity = id => this.calculatedFieldsService.deleteCalculatedField(id.id); + this.addActionDescriptors = [ + { + name: this.translate.instant('calculated-fields.create'), + icon: 'insert_drive_file', + isEnabled: () => true, + onAction: ($event) => this.getTable().addEntity($event) + }, + { + name: this.translate.instant('calculated-fields.import'), + icon: 'file_upload', + isEnabled: () => true, + onAction: () => this.importCalculatedField() + } + ]; + + this.defaultSortOrder = {property: 'name', direction: Direction.DESC}; + + const expressionColumn = new EntityTableColumn('expression', 'calculated-fields.expression', '33%', entity => entity.configuration?.expression); + expressionColumn.sortable = false; + + this.columns.push(new EntityTableColumn('name', 'common.name', '33%')); + this.columns.push(new EntityTableColumn('type', 'common.type', '50px', entity => this.translate.instant(CalculatedFieldTypeTranslations.get(entity.type)))); + this.columns.push(expressionColumn); + + this.cellActionDescriptors.push( + { + name: this.translate.instant('action.export'), + icon: 'file_download', + isEnabled: () => true, + onAction: (event$, entity) => this.exportCalculatedField(event$, entity), + }, + { + name: this.translate.instant('entity-view.events'), + icon: 'mdi:clipboard-text-clock', + isEnabled: () => true, + onAction: (_, entity) => this.openDebugEventsDialog(entity), + }, + { + name: '', + nameFunction: entity => this.getDebugConfigLabel(entity?.debugSettings), + icon: 'mdi:bug', + isEnabled: () => true, + iconFunction: ({ debugSettings }) => this.isDebugActive(debugSettings?.allEnabledUntil) || debugSettings?.failuresEnabled ? 'mdi:bug' : 'mdi:bug-outline', + onAction: ($event, entity) => this.onOpenDebugConfig($event, entity), + }, + { + name: this.translate.instant('action.edit'), + icon: 'edit', + isEnabled: () => true, + onAction: (_, entity) => this.editCalculatedField(entity), + } + ); + } + + fetchCalculatedFields(pageLink: PageLink): Observable> { + return this.calculatedFieldsService.getCalculatedFields(this.entityId, pageLink); + } + + onOpenDebugConfig($event: Event, calculatedField: CalculatedField): void { + const { debugSettings = {}, id } = calculatedField; + const additionalActionConfig = { + ...this.additionalDebugActionConfig, + action: () => this.openDebugEventsDialog(calculatedField) + }; + const { viewContainerRef } = this.getTable(); + if ($event) { + $event.stopPropagation(); + } + const trigger = $event.target as Element; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const debugStrategyPopover = this.popoverService.displayPopover(trigger, this.renderer, + viewContainerRef, EntityDebugSettingsPanelComponent, 'bottom', true, null, + { + debugLimitsConfiguration: this.calculatedFieldsDebugPerTenantLimitsConfiguration, + maxDebugModeDuration: this.maxDebugModeDuration, + entityLabel: this.translate.instant('debug-settings.calculated-field'), + additionalActionConfig, + ...debugSettings + }, + {}, + {}, {}, true); + debugStrategyPopover.tbComponentRef.instance.onSettingsApplied.pipe(takeUntilDestroyed(this.destroyRef)).subscribe((settings: EntityDebugSettings) => { + this.onDebugConfigChanged(id.id, settings); + debugStrategyPopover.hide(); + }); + } + } + + private editCalculatedField(calculatedField: CalculatedField, isDirty = false): void { + this.getCalculatedFieldDialog(calculatedField, 'action.apply', isDirty) + .subscribe((res) => { + if (res) { + this.updateData(); + } + }); + } + + private getCalculatedFieldDialog(value?: CalculatedField, buttonTitle = 'action.add', isDirty = false): Observable { + return this.dialog.open(CalculatedFieldDialogComponent, { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], + data: { + value, + buttonTitle, + entityId: this.entityId, + debugLimitsConfiguration: this.calculatedFieldsDebugPerTenantLimitsConfiguration, + tenantId: this.tenantId, + entityName: this.entityName, + additionalDebugActionConfig: this.additionalDebugActionConfig, + getTestScriptDialogFn: this.getTestScriptDialog.bind(this), + isDirty, + }, + enterAnimationDuration: isDirty ? 0 : null, + }) + .afterClosed() + .pipe(filter(Boolean)); + } + + private openDebugEventsDialog(calculatedField: CalculatedField): void { + this.dialog.open(CalculatedFieldDebugDialogComponent, { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], + data: { + tenantId: this.tenantId, + value: calculatedField, + getTestScriptDialogFn: this.getTestScriptDialog.bind(this), + } + }) + .afterClosed() + .subscribe(); + } + + private exportCalculatedField($event: Event, calculatedField: CalculatedField): void { + if ($event) { + $event.stopPropagation(); + } + this.importExportService.exportCalculatedField(calculatedField.id.id); + } + + private importCalculatedField(): void { + this.importExportService.importCalculatedField(this.entityId) + .pipe(filter(Boolean), takeUntilDestroyed(this.destroyRef)) + .subscribe(() => this.updateData()); + } + + private getDebugConfigLabel(debugSettings: EntityDebugSettings): string { + const isDebugActive = this.isDebugActive(debugSettings?.allEnabledUntil); + + if (!isDebugActive) { + return debugSettings?.failuresEnabled ? this.translate.instant('debug-settings.failures') : this.translate.instant('common.disabled'); + } else { + return this.durationLeft.transform(debugSettings?.allEnabledUntil); + } + } + + private isDebugActive(allEnabledUntil: number): boolean { + return allEnabledUntil > new Date().getTime(); + } + + private onDebugConfigChanged(id: string, debugSettings: EntityDebugSettings): void { + this.calculatedFieldsService.getCalculatedFieldById(id).pipe( + switchMap(field => this.calculatedFieldsService.saveCalculatedField({ ...field, debugSettings })), + catchError(() => of(null)), + takeUntilDestroyed(this.destroyRef), + ).subscribe(() => this.updateData()); + } + + private getTestScriptDialog(calculatedField: CalculatedField, argumentsObj?: CalculatedFieldEventArguments, openCalculatedFieldEdit = true): Observable { + const resultArguments = Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { + const type = calculatedField.configuration.arguments[key].refEntityKey.type; + acc[key] = isObject(argumentsObj) && argumentsObj.hasOwnProperty(key) + ? { ...argumentsObj[key], type } + : type === ArgumentType.Rolling ? { values: [], type } : { value: '', type, ts: new Date().getTime() }; + return acc; + }, {}); + return this.dialog.open(CalculatedFieldScriptTestDialogComponent, + { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog', 'tb-fullscreen-dialog-gt-xs'], + data: { + arguments: resultArguments, + expression: calculatedField.configuration.expression, + argumentsEditorCompleter: getCalculatedFieldArgumentsEditorCompleter(calculatedField.configuration.arguments), + argumentsHighlightRules: getCalculatedFieldArgumentsHighlights(calculatedField.configuration.arguments), + openCalculatedFieldEdit + } + }).afterClosed() + .pipe( + filter(Boolean), + tap(expression => { + if (openCalculatedFieldEdit) { + this.editCalculatedField({ entityId: this.entityId, ...calculatedField, configuration: {...calculatedField.configuration, expression } }, true) + } + }), + ); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html new file mode 100644 index 0000000000..df433bc70e --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html @@ -0,0 +1,20 @@ + +@if (calculatedFieldsTableConfig) { + +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.scss new file mode 100644 index 0000000000..3feb1e7429 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.scss @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host ::ng-deep { + tb-entities-table { + .mat-drawer-container { + background-color: white; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts new file mode 100644 index 0000000000..be6fb11480 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -0,0 +1,85 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + ChangeDetectionStrategy, + ChangeDetectorRef, + Component, + DestroyRef, + effect, + input, + Renderer2, + ViewChild, +} from '@angular/core'; +import { EntityId } from '@shared/models/id/entity-id'; +import { EntitiesTableComponent } from '@home/components/entity/entities-table.component'; +import { TranslateService } from '@ngx-translate/core'; +import { MatDialog } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { CalculatedFieldsTableConfig } from '@home/components/calculated-fields/calculated-fields-table-config'; +import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; +import { ImportExportService } from '@shared/import-export/import-export.service'; + +@Component({ + selector: 'tb-calculated-fields-table', + templateUrl: './calculated-fields-table.component.html', + styleUrls: ['./calculated-fields-table.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class CalculatedFieldsTableComponent { + + @ViewChild(EntitiesTableComponent, {static: true}) entitiesTable: EntitiesTableComponent; + + active = input(); + entityId = input(); + entityName = input(); + + calculatedFieldsTableConfig: CalculatedFieldsTableConfig; + + constructor(private calculatedFieldsService: CalculatedFieldsService, + private translate: TranslateService, + private dialog: MatDialog, + private store: Store, + private durationLeft: DurationLeftPipe, + private popoverService: TbPopoverService, + private cd: ChangeDetectorRef, + private renderer: Renderer2, + private importExportService: ImportExportService, + private destroyRef: DestroyRef) { + + effect(() => { + if (this.active()) { + this.calculatedFieldsTableConfig = new CalculatedFieldsTableConfig( + this.calculatedFieldsService, + this.translate, + this.dialog, + this.entityId(), + this.store, + this.durationLeft, + this.popoverService, + this.destroyRef, + this.renderer, + this.entityName(), + this.importExportService + ); + this.cd.markForCheck(); + } + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html new file mode 100644 index 0000000000..6bab01f976 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -0,0 +1,140 @@ + +
    +
    + + + +
    {{ 'common.name' | translate }}
    +     + +
    {{ argument.argumentName }}
    +     +     + + + {{ 'entity.entity-type' | translate }} + + +
    + @if (argument.refEntityId?.entityType === ArgumentEntityType.Tenant) { + {{ 'calculated-fields.argument-current-tenant' | translate }} + } @else if (argument.refEntityId?.id) { + {{ entityTypeTranslations.get(argument.refEntityId.entityType).type | translate }} + } @else { + {{ 'calculated-fields.argument-current' | translate }} + } +
    +     +     + + + {{ 'entity-view.target-entity' | translate }} + + +
    + @if (argument.refEntityId?.id) { + + {{ entityNameMap.get(argument.refEntityId.id) ?? '' }} + + } +
    +     +     + + + {{ 'common.type' | translate }} + + +
    {{ ArgumentTypeTranslations.get(argument.refEntityKey.type) | translate }}
    +     +     + + + {{ 'entity.key' | translate }} + + + +
    {{ argument.refEntityKey.key }}
    +     +     +     + + + +
    + + +
    +     +     + + +
    +
    + {{ 'calculated-fields.no-arguments' | translate }} +
    + @if (errorText) { + + } +
    +
    + + @if (maxArgumentsPerCF && argumentsFormArray.length >= maxArgumentsPerCF) { +
    + warning + {{ 'calculated-fields.hint.max-args' | translate }} +
    + } +
    +
    diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss new file mode 100644 index 0000000000..877a749afa --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .arguments-table { + min-height: 108px; + + &-with-error { + min-height: 150px; + } + + .mat-mdc-table { + table-layout: fixed; + } + + .key-text { + font-size: 13px; + } + } + + .max-args-warning { + .mat-icon { + color: #FAA405; + } + } + + .tb-form-table-row-cell-buttons { + --mat-badge-legacy-small-size-container-size: 8px; + --mat-badge-small-size-container-overlap-offset: -5px; + --mat-badge-small-size-text-size: 0; + } +} + +:host ::ng-deep { + .mat-mdc-standard-chip { + .mdc-evolution-chip__cell--primary, .mdc-evolution-chip__action--primary, .mdc-evolution-chip__text-label { + overflow: hidden; + } + } + + .arguments-table:not(.arguments-table-with-error) { + .mdc-data-table__row:last-child .mat-mdc-cell { + border-bottom: none; + } + } + + .arguments-table { + .mat-mdc-header-row.mat-row-select .mat-mdc-header-cell.entity-type-header { + padding: 0 28px 0 0; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts new file mode 100644 index 0000000000..e3b19503af --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -0,0 +1,278 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + AfterViewInit, + ChangeDetectorRef, + Component, + DestroyRef, + forwardRef, + Input, + OnChanges, + Renderer2, + SimpleChanges, + ViewChild, + ViewContainerRef, +} from '@angular/core'; +import { + AbstractControl, + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, +} from '@angular/forms'; +import { + ArgumentEntityType, + ArgumentType, + ArgumentTypeTranslations, + CalculatedFieldArgument, + CalculatedFieldArgumentValue, + CalculatedFieldType, +} from '@shared/models/calculated-field.models'; +import { CalculatedFieldArgumentPanelComponent } from '@home/components/calculated-fields/components/public-api'; +import { MatButton } from '@angular/material/button'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { EntityId } from '@shared/models/id/entity-id'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { getEntityDetailsPageURL, isDefined, isDefinedAndNotNull } from '@core/utils'; +import { TbPopoverComponent } from '@shared/components/popover.component'; +import { TbTableDatasource } from '@shared/components/table/table-datasource.abstract'; +import { EntityService } from '@core/http/entity.service'; +import { MatSort } from '@angular/material/sort'; +import { getCurrentAuthState } from '@core/auth/auth.selectors'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; + +@Component({ + selector: 'tb-calculated-field-arguments-table', + templateUrl: './calculated-field-arguments-table.component.html', + styleUrls: [`calculated-field-arguments-table.component.scss`], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => CalculatedFieldArgumentsTableComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => CalculatedFieldArgumentsTableComponent), + multi: true + } + ], +}) +export class CalculatedFieldArgumentsTableComponent implements ControlValueAccessor, Validator, OnChanges, AfterViewInit { + + @Input() entityId: EntityId; + @Input() tenantId: string; + @Input() entityName: string; + @Input() calculatedFieldType: CalculatedFieldType; + + @ViewChild(MatSort, { static: true }) sort: MatSort; + + errorText = ''; + argumentsFormArray = this.fb.array([]); + entityNameMap = new Map(); + sortOrder = { direction: 'asc', property: '' }; + dataSource = new CalculatedFieldArgumentDatasource(); + + readonly entityTypeTranslations = entityTypeTranslations; + readonly ArgumentTypeTranslations = ArgumentTypeTranslations; + readonly ArgumentEntityType = ArgumentEntityType; + readonly ArgumentType = ArgumentType; + readonly CalculatedFieldType = CalculatedFieldType; + readonly maxArgumentsPerCF = getCurrentAuthState(this.store).maxArgumentsPerCF; + + private popoverComponent: TbPopoverComponent; + private propagateChange: (argumentsObj: Record) => void = () => {}; + + constructor( + private fb: FormBuilder, + private popoverService: TbPopoverService, + private viewContainerRef: ViewContainerRef, + private cd: ChangeDetectorRef, + private renderer: Renderer2, + private entityService: EntityService, + private destroyRef: DestroyRef, + private store: Store + ) { + this.argumentsFormArray.valueChanges.pipe(takeUntilDestroyed()).subscribe(value => { + this.updateEntityNameMap(value); + this.updateDataSource(value); + this.propagateChange(this.getArgumentsObject(value)); + }); + } + + ngOnChanges(changes: SimpleChanges): void { + if (changes.calculatedFieldType?.previousValue + && changes.calculatedFieldType.currentValue !== changes.calculatedFieldType.previousValue) { + this.argumentsFormArray.updateValueAndValidity(); + } + } + + ngAfterViewInit(): void { + this.sort.sortChange.asObservable().pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => { + this.sortOrder.property = this.sort.active; + this.sortOrder.direction = this.sort.direction; + this.updateDataSource(this.argumentsFormArray.value); + }); + } + + registerOnChange(fn: (argumentsObj: Record) => void): void { + this.propagateChange = fn; + } + + registerOnTouched(_): void {} + + validate(): ValidationErrors | null { + this.updateErrorText(); + return this.errorText ? { argumentsFormArray: false } : null; + } + + onDelete($event: Event, index: number): void { + $event.stopPropagation(); + this.argumentsFormArray.removeAt(index); + this.argumentsFormArray.markAsDirty(); + } + + manageArgument($event: Event, matButton: MatButton, argument = {} as CalculatedFieldArgumentValue, index?: number): void { + $event?.stopPropagation(); + if (this.popoverComponent && !this.popoverComponent.tbHidden) { + this.popoverComponent.hide(); + } + const trigger = matButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const ctx = { + index, + argument, + entityId: this.entityId, + calculatedFieldType: this.calculatedFieldType, + buttonTitle: this.argumentsFormArray.at(index)?.value ? 'action.apply' : 'action.add', + tenantId: this.tenantId, + entityName: this.entityName, + usedArgumentNames: this.argumentsFormArray.value.map(({ argumentName }) => argumentName).filter(name => name !== argument.argumentName), + }; + this.popoverComponent = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, CalculatedFieldArgumentPanelComponent, isDefined(index) ? 'left' : 'right', false, null, + ctx, + {}, + {}, {}, true); + this.popoverComponent.tbComponentRef.instance.argumentsDataApplied.subscribe(({ value, index }) => { + this.popoverComponent.hide(); + const formGroup = this.fb.group(value); + if (isDefinedAndNotNull(index)) { + this.argumentsFormArray.setControl(index, formGroup); + } else { + this.argumentsFormArray.push(formGroup); + } + formGroup.markAsDirty(); + this.cd.markForCheck(); + }); + } + } + + private updateDataSource(value: CalculatedFieldArgumentValue[]): void { + const sortedValue = this.sortData(value); + this.dataSource.loadData(sortedValue); + } + + private updateErrorText(): void { + if (this.calculatedFieldType === CalculatedFieldType.SIMPLE + && this.argumentsFormArray.controls.some(control => control.value.refEntityKey.type === ArgumentType.Rolling)) { + this.errorText = 'calculated-fields.hint.arguments-simple-with-rolling'; + } else if (!this.argumentsFormArray.controls.length) { + this.errorText = 'calculated-fields.hint.arguments-empty'; + } else { + this.errorText = ''; + } + } + + private getArgumentsObject(value: CalculatedFieldArgumentValue[]): Record { + return value.reduce((acc, argumentValue) => { + const { argumentName, ...argument } = argumentValue as CalculatedFieldArgumentValue; + acc[argumentName] = argument; + return acc; + }, {} as Record); + } + + writeValue(argumentsObj: Record): void { + this.argumentsFormArray.clear(); + this.populateArgumentsFormArray(argumentsObj) + } + + getEntityDetailsPageURL(id: string, type: EntityType): string { + return getEntityDetailsPageURL(id, type); + } + + private populateArgumentsFormArray(argumentsObj: Record): void { + Object.keys(argumentsObj).forEach(key => { + const value: CalculatedFieldArgumentValue = { + ...argumentsObj[key], + argumentName: key + }; + this.argumentsFormArray.push(this.fb.group(value), { emitEvent: false }); + }); + this.argumentsFormArray.updateValueAndValidity(); + } + + private updateEntityNameMap(value: CalculatedFieldArgumentValue[]): void { + value.forEach(({ refEntityId = {}}) => { + if (refEntityId.id && !this.entityNameMap.has(refEntityId.id)) { + const { id, entityType } = refEntityId as EntityId; + this.entityService.getEntity(entityType as EntityType, id, { ignoreLoading: true, ignoreErrors: true }) + .pipe(takeUntilDestroyed(this.destroyRef)) + .subscribe(entity => this.entityNameMap.set(id, entity.name)); + } + }); + } + + private getSortValue(argument: CalculatedFieldArgumentValue, column: string): string { + switch (column) { + case 'entityType': + if (argument.refEntityId?.entityType === ArgumentEntityType.Tenant) { + return 'calculated-fields.argument-current-tenant'; + } else if (argument.refEntityId?.id) { + return entityTypeTranslations.get((argument.refEntityId)?.entityType as unknown as EntityType).type; + } else { + return 'calculated-fields.argument-current'; + } + case 'type': + return ArgumentTypeTranslations.get(argument.refEntityKey.type); + case 'key': + return argument.refEntityKey.key; + default: + return argument.argumentName; + } + } + + private sortData(data: CalculatedFieldArgumentValue[]): CalculatedFieldArgumentValue[] { + return data.sort((a, b) => { + const valA = this.getSortValue(a, this.sortOrder.property) ?? ''; + const valB = this.getSortValue(b, this.sortOrder.property) ?? ''; + return (this.sortOrder.direction === 'asc' ? 1 : -1) * valA.localeCompare(valB); + }); + } +} + +class CalculatedFieldArgumentDatasource extends TbTableDatasource { + constructor() { + super(); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html new file mode 100644 index 0000000000..d9cb7eb021 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -0,0 +1,48 @@ + +
    + +

    {{ 'calculated-fields.debugging' | translate}}

    + + +     +
    + +
    +
    + +
    +
    diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss new file mode 100644 index 0000000000..c4798d06f9 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .debug-dialog-container { + width: 1080px; + max-width: 100%; + + .debug-dialog-content { + height: 65vh; + border-radius: 0; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts new file mode 100644 index 0000000000..8618a11990 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts @@ -0,0 +1,60 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { AfterViewInit, Component, Inject, ViewChild } from '@angular/core'; +import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { Router } from '@angular/router'; +import { DialogComponent } from '@shared/components/dialog.component'; +import { CalculatedFieldEventBody, DebugEventType, EventType } from '@shared/models/event.models'; +import { EventTableComponent } from '@home/components/event/event-table.component'; +import { CalculatedFieldDebugDialogData, CalculatedFieldType } from '@shared/models/calculated-field.models'; + +@Component({ + selector: 'tb-calculated-field-debug-dialog', + styleUrls: ['calculated-field-debug-dialog.component.scss'], + templateUrl: './calculated-field-debug-dialog.component.html', +}) +export class CalculatedFieldDebugDialogComponent extends DialogComponent implements AfterViewInit { + + @ViewChild(EventTableComponent, {static: true}) eventsTable: EventTableComponent; + + readonly DebugEventType = DebugEventType; + readonly debugEventTypes = DebugEventType; + readonly EventType = EventType; + + constructor(protected store: Store, + protected router: Router, + @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldDebugDialogData, + protected dialogRef: MatDialogRef) { + super(store, router, dialogRef); + } + + ngAfterViewInit(): void { + this.eventsTable.entitiesTable.updateData(); + this.eventsTable.entitiesTable.cellActionDescriptors[0].isEnabled = () => this.data.value.type === CalculatedFieldType.SCRIPT; + } + + cancel(): void { + this.dialogRef.close(null); + } + + onDebugEventSelected(event: CalculatedFieldEventBody): void { + this.data.getTestScriptDialogFn(this.data.value, JSON.parse(event.arguments)) + .subscribe(expression => this.dialogRef.close(expression)); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html new file mode 100644 index 0000000000..7e0f87a12e --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -0,0 +1,195 @@ + +
    + +

    {{ 'entity.type-calculated-field' | translate}}

    + +
    + +     +
    +
    +
    +
    {{ 'common.general' | translate }}
    +
    + + {{ 'entity-field.title' | translate }} + + @if (fieldFormGroup.get('name').errors && fieldFormGroup.get('name').touched) { + + @if (fieldFormGroup.get('name').hasError('required')) { + {{ 'common.hint.title-required' | translate }} + } @else if (fieldFormGroup.get('name').hasError('pattern')) { + {{ 'common.hint.title-pattern' | translate }} + } @else if (fieldFormGroup.get('name').hasError('maxlength')) { + {{ 'common.hint.title-max-length' | translate }} + } + + } + + +
    + + {{ 'common.type' | translate }} + + @for (type of fieldTypes; track type) { + {{ CalculatedFieldTypeTranslations.get(type) | translate}} + } + + +
    + +
    +
    {{ 'calculated-fields.arguments' | translate }}
    + +
    +
    +
    {{ 'calculated-fields.expression' | translate }}
    + @if (fieldFormGroup.get('type').value === CalculatedFieldType.SIMPLE) { + + + @if (configFormGroup.get('expressionSIMPLE').errors && configFormGroup.get('expressionSIMPLE').touched) { + + @if (configFormGroup.get('expressionSIMPLE').hasError('required')) { + {{ 'calculated-fields.hint.expression-required' | translate }} + } @else if (configFormGroup.get('expressionSIMPLE').hasError('pattern')) { + {{ 'calculated-fields.hint.expression-invalid' | translate }} + } @else if (configFormGroup.get('expressionSIMPLE').hasError('maxLength')) { + {{ 'calculated-fields.hint.expression-max-length' | translate }} + } + + } @else { + {{ 'calculated-fields.hint.expression' | translate }} + } + + } @else { + +
    {{ 'api-usage.tbel' | translate }}
    + +     +
    + +
    + } +
    +
    +
    {{ 'calculated-fields.output' | translate }}
    +
    + + {{ 'calculated-fields.output-type' | translate }} + + @for (type of outputTypes; track type) { + {{ OutputTypeTranslations.get(type) | translate}} + } + + + @if (outputFormGroup.get('type').value === OutputType.Attribute + && (data.entityId.entityType === EntityType.DEVICE || data.entityId.entityType === EntityType.DEVICE_PROFILE)) { + + {{ 'calculated-fields.attribute-scope' | translate }} + + + {{ 'calculated-fields.server-attributes' | translate }} + + + {{ 'calculated-fields.shared-attributes' | translate }} + + + + } +
    + @if (fieldFormGroup.get('type').value === CalculatedFieldType.SIMPLE) { + + + {{ (outputFormGroup.get('type').value === OutputType.Timeseries + ? 'calculated-fields.timeseries-key' + : 'calculated-fields.attribute-key') + | translate }} + + + @if (outputFormGroup.get('name').errors && outputFormGroup.get('name').touched) { + + @if (outputFormGroup.get('name').hasError('required')) { + {{ 'common.hint.key-required' | translate }} + } @else if (outputFormGroup.get('name').hasError('pattern')) { + {{ 'common.hint.key-pattern' | translate }} + } @else if (outputFormGroup.get('name').hasError('maxlength')) { + {{ 'common.hint.key-max-length' | translate }} + } + + } + + } +
    +     +
    +
    +
    + + +
    +
    diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss new file mode 100644 index 0000000000..0e994ed825 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .dialog-container { + width: 869px; + max-width: 100%; + } + + .script-lang-chip { + line-height: 20px; + font-size: 14px; + font-weight: 500; + color: white; + border-radius: 100px; + width: 70px; + display: flex; + justify-content: center; + margin-top: 2px; + margin-right: 4px; + } +} + +:host ::ng-deep { + .expression-edit { + .ace_tb { + &.ace_calculated-field { + &-key { + color: #C52F00; + } + &-ts, &-time-window, &-values, &-value, &-func { + color: #7214D0; + } + &-start-ts, &-end-ts, &-limit { + color: #185F2A; + } + } + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts new file mode 100644 index 0000000000..c3e3d52a72 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -0,0 +1,194 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, DestroyRef, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { Router } from '@angular/router'; +import { DialogComponent } from '@shared/components/dialog.component'; +import { + CalculatedField, + CalculatedFieldConfiguration, + CalculatedFieldDialogData, + CalculatedFieldType, + CalculatedFieldTypeTranslations, + getCalculatedFieldArgumentsEditorCompleter, + getCalculatedFieldArgumentsHighlights, + OutputType, + OutputTypeTranslations +} from '@shared/models/calculated-field.models'; +import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; +import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; +import { EntityType } from '@shared/models/entity-type.models'; +import { map, startWith } from 'rxjs/operators'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { ScriptLanguage } from '@shared/models/rule-node.models'; +import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; + +@Component({ + selector: 'tb-calculated-field-dialog', + templateUrl: './calculated-field-dialog.component.html', + styleUrls: ['./calculated-field-dialog.component.scss'], +}) +export class CalculatedFieldDialogComponent extends DialogComponent { + + fieldFormGroup = this.fb.group({ + name: ['', [Validators.required, Validators.pattern(noLeadTrailSpacesRegex), Validators.maxLength(255)]], + type: [CalculatedFieldType.SIMPLE], + debugSettings: [], + configuration: this.fb.group({ + arguments: this.fb.control({}), + expressionSIMPLE: ['', [Validators.required, Validators.pattern(noLeadTrailSpacesRegex), Validators.maxLength(255)]], + expressionSCRIPT: [], + output: this.fb.group({ + name: ['', [Validators.required, Validators.pattern(noLeadTrailSpacesRegex), Validators.maxLength(255)]], + scope: [{ value: AttributeScope.SERVER_SCOPE, disabled: true }], + type: [OutputType.Timeseries] + }), + }), + }); + + functionArgs$ = this.configFormGroup.get('arguments').valueChanges + .pipe( + startWith(this.data.value?.configuration?.arguments ?? {}), + map(argumentsObj => Object.keys(argumentsObj)) + ); + + argumentsEditorCompleter$ = this.configFormGroup.get('arguments').valueChanges + .pipe( + startWith(this.data.value?.configuration?.arguments ?? {}), + map(argumentsObj => getCalculatedFieldArgumentsEditorCompleter(argumentsObj)) + ); + + argumentsHighlightRules$ = this.configFormGroup.get('arguments').valueChanges + .pipe( + startWith(this.data.value?.configuration?.arguments ?? {}), + map(argumentsObj => getCalculatedFieldArgumentsHighlights(argumentsObj)) + ); + + additionalDebugActionConfig = this.data.value?.id ? { + ...this.data.additionalDebugActionConfig, + action: () => this.data.additionalDebugActionConfig.action({ id: this.data.value.id, ...this.fromGroupValue }), + } : null; + + readonly OutputTypeTranslations = OutputTypeTranslations; + readonly OutputType = OutputType; + readonly AttributeScope = AttributeScope; + readonly EntityType = EntityType; + readonly CalculatedFieldType = CalculatedFieldType; + readonly ScriptLanguage = ScriptLanguage; + readonly fieldTypes = Object.values(CalculatedFieldType) as CalculatedFieldType[]; + readonly outputTypes = Object.values(OutputType) as OutputType[]; + readonly CalculatedFieldTypeTranslations = CalculatedFieldTypeTranslations; + + constructor(protected store: Store, + protected router: Router, + @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldDialogData, + protected dialogRef: MatDialogRef, + private calculatedFieldsService: CalculatedFieldsService, + private destroyRef: DestroyRef, + private fb: FormBuilder) { + super(store, router, dialogRef); + this.observeIsLoading(); + this.applyDialogData(); + this.observeTypeChanges(); + } + + get configFormGroup(): FormGroup { + return this.fieldFormGroup.get('configuration') as FormGroup; + } + + get outputFormGroup(): FormGroup { + return this.fieldFormGroup.get('configuration').get('output') as FormGroup; + } + + get fromGroupValue(): CalculatedField { + const { configuration, type, ...rest } = this.fieldFormGroup.value; + const { expressionSIMPLE, expressionSCRIPT, ...restConfig } = configuration; + return { configuration: { ...restConfig, type, expression: configuration['expression'+type] }, ...rest, type } as CalculatedField; + } + + cancel(): void { + this.dialogRef.close(null); + } + + add(): void { + if (this.fieldFormGroup.valid) { + this.calculatedFieldsService.saveCalculatedField({ entityId: this.data.entityId, ...(this.data.value ?? {}), ...this.fromGroupValue}) + .pipe(takeUntilDestroyed(this.destroyRef)) + .subscribe(calculatedField => this.dialogRef.close(calculatedField)); + } + } + + onTestScript(): void { + this.data.getTestScriptDialogFn(this.fromGroupValue, null, false).subscribe(expression => { + this.configFormGroup.get('expressionSCRIPT').setValue(expression); + this.configFormGroup.get('expressionSCRIPT').markAsDirty(); + }); + } + + private applyDialogData(): void { + const { configuration = {}, type = CalculatedFieldType.SIMPLE, debugSettings = { failuresEnabled: true, allEnabled: true }, ...value } = this.data.value ?? {}; + const { expression, ...restConfig } = configuration as CalculatedFieldConfiguration; + const updatedConfig = { ...restConfig , ['expression'+type]: expression }; + this.fieldFormGroup.patchValue({ configuration: updatedConfig, type, debugSettings, ...value }, {emitEvent: false}); + } + + private observeTypeChanges(): void { + this.toggleKeyByCalculatedFieldType(this.fieldFormGroup.get('type').value); + this.toggleScopeByOutputType(this.outputFormGroup.get('type').value); + + this.outputFormGroup.get('type').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(type => this.toggleScopeByOutputType(type)); + this.fieldFormGroup.get('type').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(type => this.toggleKeyByCalculatedFieldType(type)); + } + + private toggleScopeByOutputType(type: OutputType): void { + this.outputFormGroup.get('scope')[type === OutputType.Attribute? 'enable' : 'disable']({emitEvent: false}); + } + + private toggleKeyByCalculatedFieldType(type: CalculatedFieldType): void { + if (type === CalculatedFieldType.SIMPLE) { + this.outputFormGroup.get('name').enable({emitEvent: false}); + this.configFormGroup.get('expressionSIMPLE').enable({emitEvent: false}); + this.configFormGroup.get('expressionSCRIPT').disable({emitEvent: false}); + } else { + this.outputFormGroup.get('name').disable({emitEvent: false}); + this.configFormGroup.get('expressionSIMPLE').disable({emitEvent: false}); + this.configFormGroup.get('expressionSCRIPT').enable({emitEvent: false}); + } + } + + private observeIsLoading(): void { + this.isLoading$.pipe(takeUntilDestroyed()).subscribe(loading => { + if (loading) { + this.fieldFormGroup.disable({emitEvent: false}); + } else { + this.fieldFormGroup.enable({emitEvent: false}); + this.toggleScopeByOutputType(this.outputFormGroup.get('type').value); + this.toggleKeyByCalculatedFieldType(this.fieldFormGroup.get('type').value); + if (this.data.isDirty) { + this.fieldFormGroup.markAsDirty(); + } + } + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html new file mode 100644 index 0000000000..566a433545 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -0,0 +1,200 @@ + +
    +
    +
    {{ 'calculated-fields.argument-settings' | translate }}
    +
    +
    +
    {{ 'calculated-fields.argument-name' | translate }}
    + + + @if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('required')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('duplicateName')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('pattern')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('maxlength')) { + + warning + + } + +
    + +
    +
    {{ 'entity.entity-type' | translate }}
    + + + @for (type of argumentEntityTypes; track type) { + {{ ArgumentEntityTypeTranslations.get(type) | translate }} + } + + +
    + @if (ArgumentEntityTypeParamsMap.has(entityType)) { +
    +
    {{ ArgumentEntityTypeParamsMap.get(entityType).title | translate }}
    + +
    + } +     + +
    +
    {{ 'calculated-fields.argument-type' | translate }}
    + + + @for (type of argumentTypes; track type) { + {{ ArgumentTypeTranslations.get(type) | translate }} + } + + @if (refEntityKeyFormGroup.get('type').hasError('required') && refEntityKeyFormGroup.get('type').touched) { + + warning + + } + +
    + @if (entityFilter.singleEntity?.id || entityType === ArgumentEntityType.Current || entityType === ArgumentEntityType.Tenant) { + @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Attribute) { +
    +
    {{ 'calculated-fields.timeseries-key' | translate }}
    + @if (refEntityKeyFormGroup.get('type').value === ArgumentType.LatestTelemetry) { + + } @else { + + } + + + +
    + } @else { + @if (enableAttributeScopeSelection) { +
    +
    {{ 'calculated-fields.attribute-scope' | translate }}
    + + + + {{ 'calculated-fields.server-attributes' | translate }} + + + {{ 'calculated-fields.client-attributes' | translate }} + + + {{ 'calculated-fields.shared-attributes' | translate }} + + + +
    + } +
    +
    {{ 'calculated-fields.attribute-key' | translate }}
    + +
    + } + } +     + @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Rolling) { +
    +
    {{ 'calculated-fields.default-value' | translate }}
    + + + +
    + } @else { +
    +
    {{ 'calculated-fields.time-window' | translate }}
    + +
    + @if (maxDataPointsPerRollingArg) { +
    +
    {{ 'calculated-fields.limit' | translate }}
    +
    + + + + + + +
    +
    + } + } +
    +
    +
    + + +
    +
    diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss new file mode 100644 index 0000000000..773489ee60 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@use '../../../../../../../scss/constants' as constants; + +$panel-width: 520px; + +:host { + display: flex; + width: $panel-width; + max-width: 100%; + max-height: 100vh; + + .fixed-title-width { + @media #{constants.$mat-xs} { + min-width: 120px; + } + } + + .limit-field-row { + @media screen and (max-width: $panel-width) { + display: flex; + flex-direction: column; + + .fixed-title-width { + align-self: flex-start; + padding-top: 8px; + } + } + } +} + +:host ::ng-deep { + .time-interval-field { + .advanced-input { + flex-direction: column; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts new file mode 100644 index 0000000000..9d9614ba28 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -0,0 +1,246 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { ChangeDetectorRef, Component, Input, OnInit, output } from '@angular/core'; +import { TbPopoverComponent } from '@shared/components/popover.component'; +import { FormBuilder, FormControl, FormGroup, ValidatorFn, Validators } from '@angular/forms'; +import { charsWithNumRegex, noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; +import { + ArgumentEntityType, + ArgumentEntityTypeParamsMap, + ArgumentEntityTypeTranslations, + ArgumentType, + ArgumentTypeTranslations, + CalculatedFieldArgumentValue, + CalculatedFieldType, + getCalculatedFieldCurrentEntityFilter +} from '@shared/models/calculated-field.models'; +import { debounceTime, delay, distinctUntilChanged, filter } from 'rxjs/operators'; +import { EntityType } from '@shared/models/entity-type.models'; +import { AttributeScope, DataKeyType } from '@shared/models/telemetry/telemetry.models'; +import { DatasourceType } from '@shared/models/widget.models'; +import { EntityId } from '@shared/models/id/entity-id'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { EntityFilter } from '@shared/models/query/query.models'; +import { AliasFilterType } from '@shared/models/alias.models'; +import { merge } from 'rxjs'; +import { MINUTE } from '@shared/models/time/time.models'; +import { getCurrentAuthState } from '@core/auth/auth.selectors'; +import { AppState } from '@core/core.state'; +import { Store } from '@ngrx/store'; + +@Component({ + selector: 'tb-calculated-field-argument-panel', + templateUrl: './calculated-field-argument-panel.component.html', + styleUrls: ['./calculated-field-argument-panel.component.scss'] +}) +export class CalculatedFieldArgumentPanelComponent implements OnInit { + + @Input() buttonTitle: string; + @Input() index: number; + @Input() argument: CalculatedFieldArgumentValue; + @Input() entityId: EntityId; + @Input() tenantId: string; + @Input() entityName: string; + @Input() calculatedFieldType: CalculatedFieldType; + @Input() usedArgumentNames: string[]; + + argumentsDataApplied = output<{ value: CalculatedFieldArgumentValue, index: number }>(); + + readonly maxDataPointsPerRollingArg = getCurrentAuthState(this.store).maxDataPointsPerRollingArg; + readonly defaultLimit = Math.floor(this.maxDataPointsPerRollingArg / 10); + + argumentFormGroup = this.fb.group({ + argumentName: ['', [Validators.required, this.uniqNameRequired(), Validators.pattern(charsWithNumRegex), Validators.maxLength(255)]], + refEntityId: this.fb.group({ + entityType: [ArgumentEntityType.Current], + id: [''] + }), + refEntityKey: this.fb.group({ + type: [ArgumentType.LatestTelemetry, [Validators.required]], + key: [''], + scope: [{ value: AttributeScope.SERVER_SCOPE, disabled: true }, [Validators.required]], + }), + defaultValue: ['', [Validators.pattern(noLeadTrailSpacesRegex)]], + limit: [{ value: this.defaultLimit, disabled: !this.maxDataPointsPerRollingArg }], + timeWindow: [MINUTE * 15], + }); + + argumentTypes: ArgumentType[]; + entityFilter: EntityFilter; + + readonly argumentEntityTypes = Object.values(ArgumentEntityType) as ArgumentEntityType[]; + readonly ArgumentEntityTypeTranslations = ArgumentEntityTypeTranslations; + readonly ArgumentType = ArgumentType; + readonly DataKeyType = DataKeyType; + readonly EntityType = EntityType; + readonly datasourceType = DatasourceType; + readonly ArgumentTypeTranslations = ArgumentTypeTranslations; + readonly AttributeScope = AttributeScope; + readonly ArgumentEntityType = ArgumentEntityType; + readonly ArgumentEntityTypeParamsMap = ArgumentEntityTypeParamsMap; + + private currentEntityFilter: EntityFilter; + + constructor( + private fb: FormBuilder, + private cd: ChangeDetectorRef, + private popover: TbPopoverComponent, + private store: Store + ) { + this.observeEntityFilterChanges(); + this.observeEntityTypeChanges() + this.observeEntityKeyChanges(); + this.observeUpdatePosition(); + } + + get entityType(): ArgumentEntityType { + return this.argumentFormGroup.get('refEntityId').get('entityType').value; + } + + get refEntityIdFormGroup(): FormGroup { + return this.argumentFormGroup.get('refEntityId') as FormGroup; + } + + get refEntityKeyFormGroup(): FormGroup { + return this.argumentFormGroup.get('refEntityKey') as FormGroup; + } + + get enableAttributeScopeSelection(): boolean { + return this.entityType === ArgumentEntityType.Device + || (this.entityType === ArgumentEntityType.Current + && (this.entityId.entityType === EntityType.DEVICE || this.entityId.entityType === EntityType.DEVICE_PROFILE)) + } + + ngOnInit(): void { + this.argumentFormGroup.patchValue(this.argument, {emitEvent: false}); + this.currentEntityFilter = getCalculatedFieldCurrentEntityFilter(this.entityName, this.entityId); + this.updateEntityFilter(this.argument.refEntityId?.entityType, true); + this.toggleByEntityKeyType(this.argument.refEntityKey?.type); + this.setInitialEntityKeyType(); + + this.argumentTypes = Object.values(ArgumentType) + .filter(type => type !== ArgumentType.Rolling || this.calculatedFieldType === CalculatedFieldType.SCRIPT); + } + + saveArgument(): void { + const { refEntityId, ...restConfig } = this.argumentFormGroup.value; + const value = (refEntityId.entityType === ArgumentEntityType.Current ? restConfig : { refEntityId, ...restConfig }) as CalculatedFieldArgumentValue; + if (refEntityId.entityType === ArgumentEntityType.Tenant) { + refEntityId.id = this.tenantId; + } + this.argumentsDataApplied.emit({ value, index: this.index }); + } + + cancel(): void { + this.popover.hide(); + } + + private toggleByEntityKeyType(type: ArgumentType): void { + const isAttribute = type === ArgumentType.Attribute; + const isRolling = type === ArgumentType.Rolling; + this.argumentFormGroup.get('refEntityKey').get('scope')[isAttribute? 'enable' : 'disable']({ emitEvent: false }); + this.argumentFormGroup.get('limit')[isRolling? 'enable' : 'disable']({ emitEvent: false }); + this.argumentFormGroup.get('timeWindow')[isRolling? 'enable' : 'disable']({ emitEvent: false }); + this.argumentFormGroup.get('defaultValue')[isRolling? 'disable' : 'enable']({ emitEvent: false }); + } + + private updateEntityFilter(entityType: ArgumentEntityType = ArgumentEntityType.Current, onInit = false): void { + let entityFilter: EntityFilter; + switch (entityType) { + case ArgumentEntityType.Current: + entityFilter = this.currentEntityFilter; + break; + case ArgumentEntityType.Tenant: + entityFilter = { + type: AliasFilterType.singleEntity, + singleEntity: { + id: this.tenantId, + entityType: EntityType.TENANT + }, + }; + break; + default: + entityFilter = { + type: AliasFilterType.singleEntity, + singleEntity: this.argumentFormGroup.get('refEntityId').value as unknown as EntityId, + }; + } + if (!onInit) { + this.argumentFormGroup.get('refEntityKey').get('key').setValue(''); + } + this.entityFilter = entityFilter; + this.cd.markForCheck(); + } + + private observeEntityFilterChanges(): void { + merge( + this.refEntityIdFormGroup.get('entityType').valueChanges, + this.refEntityKeyFormGroup.get('type').valueChanges, + this.refEntityIdFormGroup.get('id').valueChanges.pipe(filter(Boolean)), + this.refEntityKeyFormGroup.get('scope').valueChanges, + ) + .pipe(debounceTime(50), takeUntilDestroyed()) + .subscribe(() => this.updateEntityFilter(this.entityType)); + } + + private observeEntityTypeChanges(): void { + this.argumentFormGroup.get('refEntityId').get('entityType').valueChanges + .pipe(distinctUntilChanged(), takeUntilDestroyed()) + .subscribe(type => { + this.argumentFormGroup.get('refEntityId').get('id').setValue(''); + this.argumentFormGroup.get('refEntityId') + .get('id')[type === ArgumentEntityType.Tenant || type === ArgumentEntityType.Current ? 'disable' : 'enable'](); + if (!this.enableAttributeScopeSelection) { + this.refEntityKeyFormGroup.get('scope').setValue(AttributeScope.SERVER_SCOPE); + } + }); + } + + private uniqNameRequired(): ValidatorFn { + return (control: FormControl) => { + const newName = control.value.trim().toLowerCase(); + const isDuplicate = this.usedArgumentNames?.some(name => name.toLowerCase() === newName); + + return isDuplicate ? { duplicateName: true } : null; + }; + } + + private observeEntityKeyChanges(): void { + this.argumentFormGroup.get('refEntityKey').get('type').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(type => this.toggleByEntityKeyType(type)); + } + + private setInitialEntityKeyType(): void { + if (this.calculatedFieldType === CalculatedFieldType.SIMPLE && this.argument.refEntityKey?.type === ArgumentType.Rolling) { + const typeControl = this.argumentFormGroup.get('refEntityKey').get('type'); + typeControl.setValue(null); + typeControl.markAsTouched(); + } + } + + private observeUpdatePosition(): void { + merge( + this.refEntityIdFormGroup.get('entityType').valueChanges, + this.refEntityKeyFormGroup.get('type').valueChanges, + this.argumentFormGroup.get('timeWindow').valueChanges, + this.refEntityIdFormGroup.get('id').valueChanges.pipe(filter(Boolean)), + ) + .pipe(delay(50), takeUntilDestroyed()) + .subscribe(() => this.popover.updatePosition()); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts new file mode 100644 index 0000000000..9e3c52bc4f --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts @@ -0,0 +1,21 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +export * from './dialog/calculated-field-dialog.component'; +export * from './arguments-table/calculated-field-arguments-table.component'; +export * from './panel/calculated-field-argument-panel.component'; +export * from './debug-dialog/calculated-field-debug-dialog.component'; +export * from './test-dialog/calculated-field-script-test-dialog.component'; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html new file mode 100644 index 0000000000..f393a9130b --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html @@ -0,0 +1,58 @@ + +
    +
    {{ 'calculated-fields.arguments' | translate }}
    +
    +
    +
    {{ 'common.name' | translate }}
    +
    {{ 'common.type' | translate }}
    +
    {{ 'common.data' | translate }}
    +
    +
    + @for (group of argumentsFormArray.controls; track group) { +
    + + + + + + + {{ ArgumentTypeTranslations.get(argumentsTypeMap.get(group.get('argumentName').value)) | translate }} + + + +
    + @if (argumentsTypeMap.get(group.get('argumentName').value) === ArgumentType.Rolling) { + + + + } @else { + + + + + } + +
    +
    + } +
    +
    +
    diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.scss new file mode 100644 index 0000000000..cbca3002aa --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.scss @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@use '../../../../../../../scss/constants' as constants; + +:host { + .tb-form-table { + min-width: 700px; + } +} + +:host::ng-deep { + .tb-form-table-row { + .argument-value { + .tb-value-type.row { + width: 120px; + min-width: 120px; + } + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts new file mode 100644 index 0000000000..7c5580f11a --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts @@ -0,0 +1,146 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input } from '@angular/core'; +import { + ControlValueAccessor, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + Validator, + ValidationErrors, + FormBuilder, + FormGroup +} from '@angular/forms'; +import { PageComponent } from '@shared/components/page.component'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { entityTypeTranslations } from '@shared/models/entity-type.models'; +import { + ArgumentType, + ArgumentTypeTranslations, + CalculatedFieldArgumentEventValue, + CalculatedFieldRollingTelemetryArgumentValue, + CalculatedFieldSingleArgumentValue, + CalculatedFieldEventArguments, + CalculatedFieldType +} from '@shared/models/calculated-field.models'; +import { + JsonObjectEditDialogComponent, + JsonObjectEditDialogData +} from '@shared/components/dialog/json-object-edit-dialog.component'; +import { filter } from 'rxjs/operators'; +import { MatDialog } from '@angular/material/dialog'; + +@Component({ + selector: 'tb-calculated-field-test-arguments', + templateUrl: './calculated-field-test-arguments.component.html', + styleUrls: ['./calculated-field-test-arguments.component.scss'], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => CalculatedFieldTestArgumentsComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => CalculatedFieldTestArgumentsComponent), + multi: true, + } + ] +}) +export class CalculatedFieldTestArgumentsComponent extends PageComponent implements ControlValueAccessor, Validator { + + @Input() argumentsTypeMap: Map; + + argumentsFormArray = this.fb.array([]); + + readonly entityTypeTranslations = entityTypeTranslations; + readonly ArgumentTypeTranslations = ArgumentTypeTranslations; + readonly ArgumentType = ArgumentType; + readonly CalculatedFieldType = CalculatedFieldType; + + private propagateChange: (value: CalculatedFieldEventArguments) => void = () => {}; + + constructor(private fb: FormBuilder, private dialog: MatDialog) { + super(); + this.argumentsFormArray.valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(() => this.propagateChange(this.getValue())); + } + + registerOnChange(propagateChange: (value: CalculatedFieldEventArguments) => void): void { + this.propagateChange = propagateChange; + } + + registerOnTouched(_): void { + } + + writeValue(argumentsObj: CalculatedFieldEventArguments): void { + this.argumentsFormArray.clear(); + Object.keys(argumentsObj).forEach(key => { + const value = { ...argumentsObj[key], argumentName: key } as CalculatedFieldArgumentEventValue; + this.argumentsFormArray.push(this.argumentsTypeMap.get(key) === ArgumentType.Rolling + ? this.getRollingArgumentFormGroup(value as CalculatedFieldRollingTelemetryArgumentValue) + : this.getSimpleArgumentFormGroup(value as CalculatedFieldSingleArgumentValue) + ); + }); + } + + validate(): ValidationErrors | null { + return this.argumentsFormArray.valid ? null : { arguments: { valid: false } }; + } + + openEditJSONDialog(group: FormGroup): void { + this.dialog.open(JsonObjectEditDialogComponent, { + disableClose: true, + height: '760px', + maxHeight: '70vh', + minWidth: 'min(700px, 100%)', + panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], + data: { + jsonValue: this.argumentsTypeMap.get(group.get('argumentName').value) === ArgumentType.Rolling ? group.value.rollingJson : group.value, + required: true, + fillHeight: true + } + }).afterClosed() + .pipe(filter(Boolean)) + .subscribe(result => this.argumentsTypeMap.get(group.get('argumentName').value) === ArgumentType.Rolling + ? group.get('rollingJson').patchValue({ values: (result as CalculatedFieldRollingTelemetryArgumentValue).values, timeWindow: (result as CalculatedFieldRollingTelemetryArgumentValue).timeWindow }) + : group.patchValue({ ts: (result as CalculatedFieldSingleArgumentValue).ts, value: (result as CalculatedFieldSingleArgumentValue).value }) ); + } + + private getSimpleArgumentFormGroup({ argumentName, ts, value }: CalculatedFieldSingleArgumentValue): FormGroup { + return this.fb.group({ + argumentName: [{ value: argumentName, disabled: true}], + ts: [ts], + value: [value] + }) as FormGroup; + } + + private getRollingArgumentFormGroup({ argumentName, timeWindow, values }: CalculatedFieldRollingTelemetryArgumentValue): FormGroup { + return this.fb.group({ + argumentName: [{ value: argumentName, disabled: true }], + rollingJson: [{ values: values ?? [], timeWindow: timeWindow ?? {} }] + }) as FormGroup; + } + + private getValue(): CalculatedFieldEventArguments { + return this.argumentsFormArray.getRawValue().reduce((acc, rowItem) => { + const { argumentName, rollingJson = {}, ...value } = rowItem; + acc[argumentName] = { ...rollingJson, ...value }; + return acc; + }, {}); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html new file mode 100644 index 0000000000..cda99f6c09 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html @@ -0,0 +1,103 @@ + +
    + +

    {{ 'calculated-fields.test-script-function' | translate }} ({{ 'api-usage.tbel' | translate }})

    + +     +
    +
    +
    +
    +
    +
    + {{ 'calculated-fields.expression' | translate }} +
    + +
    +
    +
    +
    +
    +
    + {{ 'calculated-fields.arguments' | translate }} +
    + +
    +
    +
    +
    +
    + common.output +
    + +
    +
    +
    +
    +
    +
    +
    + + + + +
    +
    diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss new file mode 100644 index 0000000000..2187b47e0d --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .test-dialog-container { + .block-label { + padding: 4px; + color: #00acc1; + background: rgba(220, 220, 220, .35); + border-radius: 5px; + } + + .test-block-content { + padding-top: 5px; + padding-left: 5px; + border: 1px solid #c0c0c0; + overflow: scroll; + } + + .block-label-container { + position: absolute; + z-index: 10; + font-size: 12px; + font-weight: bold; + + &.left { + right: 112px; + top: 9px; + } + + &.right-bottom { + right: 40px; + top: 6px; + } + + &.right-top { + right: 8px; + top: 2px; + } + } + } +} + +:host::ng-deep { + .test-dialog-container { + .gutter { + background-color: #eee; + background-repeat: no-repeat; + background-position: 50%; + } + + .gutter.gutter-horizontal { + cursor: col-resize; + background-image: url("../../../../../../../assets/split.js/grips/horizontal.png"); + } + + .gutter.gutter-vertical { + cursor: row-resize; + background-image: url("../../../../../../../assets/split.js/grips/vertical.png"); + } + } + + .expression-edit { + .ace_tb { + &.ace_calculated-field { + &-key { + color: #C52F00; + } + &-ts, &-time-window, &-values, &-value, &-func { + color: #7214D0; + } + &-start-ts, &-end-ts, &-limit { + color: #185F2A; + } + } + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts new file mode 100644 index 0000000000..78c3b17bcb --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts @@ -0,0 +1,228 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + AfterViewInit, + Component, + DestroyRef, + ElementRef, + Inject, + OnDestroy, + ViewChild, +} from '@angular/core'; +import { MAT_DIALOG_DATA, MatDialog, MatDialogRef } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormBuilder, Validators } from '@angular/forms'; +import { NEVER, Observable, of, switchMap } from 'rxjs'; +import { Router } from '@angular/router'; +import { DialogComponent } from '@shared/components/dialog.component'; +import { ContentType } from '@shared/models/constants'; +import { JsonContentComponent } from '@shared/components/json-content.component'; +import { ScriptLanguage } from '@shared/models/rule-node.models'; +import { ActionNotificationShow } from '@core/notification/notification.actions'; +import { beautifyJs } from '@shared/models/beautify.models'; +import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { filter } from 'rxjs/operators'; +import { + ArgumentType, + CalculatedFieldEventArguments, + CalculatedFieldTestScriptDialogData, + TestArgumentTypeMap +} from '@shared/models/calculated-field.models'; + +@Component({ + selector: 'tb-calculated-field-script-test-dialog', + templateUrl: './calculated-field-script-test-dialog.component.html', + styleUrls: ['./calculated-field-script-test-dialog.component.scss'], +}) +export class CalculatedFieldScriptTestDialogComponent extends DialogComponent implements AfterViewInit, OnDestroy { + + @ViewChild('leftPanel', {static: true}) leftPanelElmRef: ElementRef; + @ViewChild('rightPanel', {static: true}) rightPanelElmRef: ElementRef; + @ViewChild('topRightPanel', {static: true}) topRightPanelElmRef: ElementRef; + @ViewChild('bottomRightPanel', {static: true}) bottomRightPanelElmRef: ElementRef; + @ViewChild('testScriptContainer', {static: true}) testScriptContainer: ElementRef; + + @ViewChild('expressionContent', {static: true}) expressionContent: JsonContentComponent; + + calculatedFieldScriptTestFormGroup = this.fb.group({ + expression: ['', Validators.required], + arguments: [], + output: [] + }); + argumentsTypeMap = new Map(); + + readonly ContentType = ContentType; + readonly ScriptLanguage = ScriptLanguage; + readonly functionArgs = Object.keys(this.data.arguments); + + private testScriptResize: ResizeObserver; + private splitObjects: SplitObject[] = []; + + constructor(protected store: Store, + protected router: Router, + @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldTestScriptDialogData, + protected dialogRef: MatDialogRef, + private dialog: MatDialog, + private fb: FormBuilder, + private destroyRef: DestroyRef, + private calculatedFieldService: CalculatedFieldsService) { + super(store, router, dialogRef); + beautifyJs(this.data.expression, {indent_size: 4}).pipe(filter(Boolean), takeUntilDestroyed()).subscribe( + (res) => this.calculatedFieldScriptTestFormGroup.get('expression').patchValue(res, {emitEvent: false}) + ); + this.calculatedFieldScriptTestFormGroup.get('arguments').patchValue(this.getArgumentsValue()); + } + + ngAfterViewInit(): void { + this.observeResize(); + } + + ngOnDestroy(): void { + super.ngOnDestroy(); + this.testScriptResize.disconnect(); + } + + cancel(): void { + this.dialogRef.close(null); + } + + onTestScript(): void { + this.testScript() + .pipe( + switchMap(output => beautifyJs(output, {indent_size: 4})), + takeUntilDestroyed(this.destroyRef) + ) + .subscribe(output => this.calculatedFieldScriptTestFormGroup.get('output').setValue(output)); + } + + save(): void { + this.testScript(true).pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => { + this.calculatedFieldScriptTestFormGroup.get('expression').markAsPristine(); + this.dialogRef.close(this.calculatedFieldScriptTestFormGroup.get('expression').value); + }); + } + + private testScript(onSave = false): Observable { + if (this.checkInputParamErrors()) { + return this.calculatedFieldService.testScript({ + expression: this.calculatedFieldScriptTestFormGroup.get('expression').value, + arguments: this.getTestArguments() + }).pipe( + switchMap(result => { + if (result.error) { + this.store.dispatch(new ActionNotificationShow( + { + message: result.error, + type: 'error' + })); + return NEVER; + } else { + if (onSave && this.data.openCalculatedFieldEdit) { + this.dialog.closeAll(); + } + return of(result.output); + } + }), + ); + } else { + return NEVER; + } + } + + private checkInputParamErrors(): boolean { + this.expressionContent.validateOnSubmit(); + return !this.calculatedFieldScriptTestFormGroup.get('expression').invalid; + } + + private observeResize(): void { + this.testScriptResize = new ResizeObserver(() => { + this.updateSizes(); + }); + + this.testScriptResize.observe(this.testScriptContainer.nativeElement); + } + + private updateSizes(): void { + this.initSplitLayout(this.testScriptContainer.nativeElement.clientWidth <= 960); + } + + private getTestArguments(): CalculatedFieldEventArguments { + const argumentsValue = this.calculatedFieldScriptTestFormGroup.get('arguments').value; + return Object.keys(argumentsValue) + .reduce((acc, key) => { + acc[key] = argumentsValue[key]; + acc[key].type = TestArgumentTypeMap.get(this.argumentsTypeMap.get(key)); + return acc; + }, {}); + } + + private getArgumentsValue(): CalculatedFieldEventArguments { + return Object.keys(this.data.arguments) + .reduce((acc, key) => { + const { type, ...argumentObj } = this.data.arguments[key]; + this.argumentsTypeMap.set(key, type); + acc[key] = argumentObj; + return acc; + }, {}); + } + + private initSplitLayout(smallMode = false): void { + const [leftPanel, rightPanel, topRightPanel, bottomRightPanel] = [ + this.leftPanelElmRef.nativeElement, + this.rightPanelElmRef.nativeElement, + this.topRightPanelElmRef.nativeElement, + this.bottomRightPanelElmRef.nativeElement + ] as unknown as string[]; + + this.splitObjects.forEach(obj => obj.destroy()); + this.splitObjects = []; + + if (smallMode) { + this.splitObjects.push( + Split([leftPanel, rightPanel], { + sizes: [33, 67], + gutterSize: 8, + cursor: 'row-resize', + direction: 'vertical' + }), + Split([topRightPanel, bottomRightPanel], { + sizes: [50, 50], + gutterSize: 8, + cursor: 'row-resize', + direction: 'vertical' + }), + ); + } else { + this.splitObjects.push( + Split([leftPanel, rightPanel], { + sizes: [50, 50], + gutterSize: 8, + cursor: 'col-resize' + }), + Split([topRightPanel, bottomRightPanel], { + sizes: [50, 50], + gutterSize: 8, + cursor: 'row-resize', + direction: 'vertical' + }) + ); + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.ts b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.ts index 26f0c1f642..852da5f7bf 100644 --- a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.ts @@ -32,7 +32,7 @@ import { EntityDebugSettingsPanelComponent } from './entity-debug-settings-panel import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { BehaviorSubject, of, shareReplay, timer } from 'rxjs'; import { SECOND, MINUTE } from '@shared/models/time/time.models'; -import { EntityDebugSettings } from '@shared/models/entity.models'; +import { AdditionalDebugActionConfig, EntityDebugSettings } from '@shared/models/entity.models'; import { map, switchMap, takeWhile } from 'rxjs/operators'; import { getCurrentAuthState } from '@core/auth/auth.selectors'; import { AppState } from '@core/core.state'; @@ -61,6 +61,7 @@ export class EntityDebugSettingsButtonComponent implements ControlValueAccessor @Input() debugLimitsConfiguration: string; @Input() entityLabel: string; + @Input() additionalActionConfig: AdditionalDebugActionConfig; debugSettingsFormGroup = this.fb.group({ failuresEnabled: [false], @@ -133,11 +134,11 @@ export class EntityDebugSettingsButtonComponent implements ControlValueAccessor ...debugSettings, maxDebugModeDuration: this.maxDebugModeDuration, debugLimitsConfiguration: this.debugLimitsConfiguration, - entityLabel: this.entityLabel + entityLabel: this.entityLabel, + additionalActionConfig: this.additionalActionConfig, }, {}, {}, {}, true); - debugStrategyPopover.tbComponentRef.instance.popover = debugStrategyPopover; debugStrategyPopover.tbComponentRef.instance.onSettingsApplied.subscribe((settings: EntityDebugSettings) => { this.debugSettingsFormGroup.patchValue(settings); this.cd.markForCheck(); diff --git a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html index f7283f0eac..cf8c0ae14d 100644 --- a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html @@ -48,20 +48,32 @@ -
    - - +
    +
    + @if (additionalActionConfig) { + + } +
    +
    + + +
    diff --git a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.ts b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.ts index 5b9060704f..5fd3613296 100644 --- a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.ts @@ -21,7 +21,7 @@ import { Component, EventEmitter, Input, - OnInit + OnInit, } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { TbPopoverComponent } from '@shared/components/popover.component'; @@ -32,7 +32,7 @@ import { SECOND } from '@shared/models/time/time.models'; import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; import { of, shareReplay, timer } from 'rxjs'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; -import { EntityDebugSettings } from '@shared/models/entity.models'; +import { AdditionalDebugActionConfig, EntityDebugSettings } from '@shared/models/entity.models'; import { distinctUntilChanged, map, startWith, switchMap, takeWhile } from 'rxjs/operators'; @Component({ @@ -48,13 +48,13 @@ import { distinctUntilChanged, map, startWith, switchMap, takeWhile } from 'rxjs }) export class EntityDebugSettingsPanelComponent extends PageComponent implements OnInit { - @Input() popover: TbPopoverComponent; @Input({ transform: booleanAttribute }) failuresEnabled = false; @Input({ transform: booleanAttribute }) allEnabled = false; @Input() entityLabel: string; @Input() allEnabledUntil = 0; @Input() maxDebugModeDuration: number; @Input() debugLimitsConfiguration: string; + @Input() additionalActionConfig: AdditionalDebugActionConfig; onFailuresControl = this.fb.control(false); debugAllControl = this.fb.control(false); @@ -82,7 +82,8 @@ export class EntityDebugSettingsPanelComponent extends PageComponent implements onSettingsApplied = new EventEmitter(); constructor(private fb: FormBuilder, - private cd: ChangeDetectorRef) { + private cd: ChangeDetectorRef, + private popover: TbPopoverComponent) { super(); this.debugAllControl.valueChanges.pipe( @@ -107,7 +108,7 @@ export class EntityDebugSettingsPanelComponent extends PageComponent implements } onCancel(): void { - this.popover?.hide(); + this.popover.hide(); } onApply(): void { diff --git a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.html b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.html index 0523539539..4f53d5f8a3 100644 --- a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.html +++ b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.html @@ -183,6 +183,7 @@ [class.lt-lg:!hidden]="column.mobileHide" *matCellDef="let entity; let row = index" [matTooltip]="cellTooltip(entity, column, row)" + #cellMatTooltip="matTooltip" matTooltipPosition="above" [style]="cellStyle(entity, column, row)"> @@ -209,6 +210,8 @@ [copyText]="column.actionCell.onAction(null, entity)" tooltipText="{{ column.actionCell.nameFunction ? column.actionCell.nameFunction(entity) : column.actionCell.name }}" tooltipPosition="above" + (mouseover)="cellMatTooltip.hide()" + (mouseleave)="cellMatTooltip.show()" [icon]="column.actionCell.icon" [style]="column.actionCell.style"> diff --git a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts index 5d171555a0..4be96f5fdd 100644 --- a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts @@ -26,7 +26,8 @@ import { OnDestroy, OnInit, SimpleChanges, - ViewChild + ViewChild, + ViewContainerRef, } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { Store } from '@ngrx/store'; @@ -141,7 +142,8 @@ export class EntitiesTableComponent extends PageComponent implements IEntitiesTa private router: Router, private elementRef: ElementRef, private fb: FormBuilder, - private zone: NgZone) { + private zone: NgZone, + public viewContainerRef: ViewContainerRef) { super(store); } @@ -687,7 +689,7 @@ export class EntitiesTableComponent extends PageComponent implements IEntitiesTa } cellTooltip(entity: BaseData, column: EntityColumn>, row: number) { - if (column instanceof EntityTableColumn) { + if (column instanceof EntityTableColumn || column instanceof EntityLinkTableColumn) { const col = this.entitiesTableConfig.columns.indexOf(column); const index = row * this.entitiesTableConfig.columns.length + col; let res = this.cellTooltipCache[index]; diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 772ef38ab8..6c334d2a5e 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -18,6 +18,7 @@ import { CellActionDescriptorType, DateEntityTableColumn, EntityActionTableColumn, + EntityLinkTableColumn, EntityTableColumn, EntityTableConfig } from '@home/models/entity/entities-table-config.models'; @@ -29,7 +30,7 @@ import { MatDialog } from '@angular/material/dialog'; import { EntityId } from '@shared/models/id/entity-id'; import { EventService } from '@app/core/http/event.service'; import { EventTableHeaderComponent } from '@home/components/event/event-table-header.component'; -import { EntityTypeResource } from '@shared/models/entity-type.models'; +import { EntityType, EntityTypeResource } from '@shared/models/entity-type.models'; import { fromEvent, Observable } from 'rxjs'; import { PageData } from '@shared/models/page/page-data'; import { Direction } from '@shared/models/page/sort-order'; @@ -39,7 +40,7 @@ import { EventContentDialogComponent, EventContentDialogData } from '@home/components/event/event-content-dialog.component'; -import { isEqual, sortObjectKeys } from '@core/utils'; +import { getEntityDetailsPageURL, isEqual, sortObjectKeys } from '@core/utils'; import { DAY, historyInterval, MINUTE } from '@shared/models/time/time.models'; import { Overlay, OverlayConfig, OverlayRef } from '@angular/cdk/overlay'; import { ChangeDetectorRef, EventEmitter, Injector, StaticProvider, ViewContainerRef } from '@angular/core'; @@ -355,6 +356,89 @@ export class EventTableConfig extends EntityTableConfig { '48px') ); break; + case DebugEventType.DEBUG_CALCULATED_FIELD: + this.columns[0].width = '80px'; + this.columns[1].width = '100px'; + this.columns.push( + new EntityLinkTableColumn('entityId', 'event.entity-id', '100px', + (entity) => `${entity.body.entityId.substring(0, 8)}…`, + (entity) => getEntityDetailsPageURL(entity.body.entityId, entity.body.entityType as EntityType), + false, + () => ({padding: '0 12px 0 0'}), + () => ({padding: '0 12px 0 0'}), + (entity) => entity.body.entityId, + { + name: this.translate.instant('event.copy-entity-id'), + icon: 'content_paste', + style: { + padding: '4px', + 'font-size': '16px', + color: 'rgba(0,0,0,.87)' + }, + isEnabled: () => true, + onAction: ($event, entity) => entity.body.entityId, + type: CellActionDescriptorType.COPY_BUTTON + } + ), + new EntityTableColumn('messageId', 'event.message-id', '100px', + (entity) => entity.body.msgId ? `${entity.body.msgId?.substring(0, 8)}…` : '-', + () => ({padding: '0 12px 0 0'}), + false, + () => ({padding: '0 12px 0 0'}), + (entity) => entity.body.msgId, + false, + { + name: this.translate.instant('event.copy-message-id'), + icon: 'content_paste', + style: { + padding: '4px', + 'font-size': '16px', + color: 'rgba(0,0,0,.87)' + }, + isEnabled: (entity) => !!entity.body.msgId, + onAction: (_, entity) => entity.body.msgId, + type: CellActionDescriptorType.COPY_BUTTON + } + ), + new EntityTableColumn('messageType', 'event.message-type', '100px', + (entity) => entity.body.msgType ?? '-', + () => ({padding: '0 12px 0 0'}), + false, + () => ({padding: '0 12px 0 0'}), + (entity) => entity.body.msgType, + ), + new EntityActionTableColumn('arguments', 'event.arguments', + { + name: this.translate.instant('action.view'), + icon: 'more_horiz', + isEnabled: (entity) => entity.body.arguments !== undefined, + onAction: ($event, entity) => this.showContent($event, entity.body.arguments, + 'event.arguments', ContentType.JSON, true) + }, + '48px' + ), + new EntityActionTableColumn('result', 'event.result', + { + name: this.translate.instant('action.view'), + icon: 'more_horiz', + isEnabled: (entity) => entity.body.result !== undefined, + onAction: ($event, entity) => this.showContent($event, entity.body.result, + 'event.result', ContentType.JSON, true) + }, + '48px' + ), + new EntityActionTableColumn('error', 'event.error', + { + name: this.translate.instant('action.view'), + icon: 'more_horiz', + isEnabled: (entity) => entity.body.error && entity.body.error.length > 0, + onAction: ($event, entity) => this.showContent($event, entity.body.error, + 'event.error') + }, + '48px' + ) + ); + break; } if (updateTableColumns) { this.getTable().columnsUpdated(true); @@ -376,6 +460,14 @@ export class EventTableConfig extends EntityTableConfig { }); } break; + case DebugEventType.DEBUG_CALCULATED_FIELD: + this.cellActionDescriptors.push({ + name: this.translate.instant('common.test-with-this-message', {test: this.translate.instant(this.testButtonLabel)}), + icon: 'bug_report', + isEnabled: () => true, + onAction: (_, entity) => this.debugEventSelected.next(entity.body) + }); + break; } this.getTable()?.cellActionDescriptorsUpdated(); } @@ -446,6 +538,17 @@ export class EventTableConfig extends EntityTableConfig { {key: 'errorStr', title: 'event.error'} ); break; + case DebugEventType.DEBUG_CALCULATED_FIELD: + this.filterColumns.push( + {key: 'entityId', title: 'event.entity-id'}, + {key: 'msgId', title: 'event.message-id'}, + {key: 'msgType', title: 'event.message-type'}, + {key: 'arguments', title: 'event.arguments'}, + {key: 'result', title: 'event.result'}, + {key: 'isError', title: 'event.error'}, + {key: 'errorStr', title: 'event.error'} + ); + break; } } diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 7fdcccad64..e680232129 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -183,6 +183,27 @@ import { } from '@home/components/dashboard-page/layout/select-dashboard-breakpoint.component'; import { EntityChipsComponent } from '@home/components/entity/entity-chips.component'; import { DashboardViewComponent } from '@home/components/dashboard-view/dashboard-view.component'; +import { CalculatedFieldsTableComponent } from '@home/components/calculated-fields/calculated-fields-table.component'; +import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; +import { CalculatedFieldDialogComponent } from '@home/components/calculated-fields/components/dialog/calculated-field-dialog.component'; +import { + EntityDebugSettingsButtonComponent +} from '@home/components/entity/debug/entity-debug-settings-button.component'; +import { + CalculatedFieldArgumentsTableComponent +} from '@home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component'; +import { + CalculatedFieldArgumentPanelComponent +} from '@home/components/calculated-fields/components/panel/calculated-field-argument-panel.component'; +import { + CalculatedFieldDebugDialogComponent +} from '@home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component'; +import { + CalculatedFieldScriptTestDialogComponent +} from '@home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component'; +import { + CalculatedFieldTestArgumentsComponent +} from '@home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component'; @NgModule({ declarations: @@ -326,7 +347,14 @@ import { DashboardViewComponent } from '@home/components/dashboard-view/dashboar RateLimitsDetailsDialogComponent, SendNotificationButtonComponent, EntityChipsComponent, - DashboardViewComponent + DashboardViewComponent, + CalculatedFieldsTableComponent, + CalculatedFieldDialogComponent, + CalculatedFieldArgumentsTableComponent, + CalculatedFieldArgumentPanelComponent, + CalculatedFieldDebugDialogComponent, + CalculatedFieldScriptTestDialogComponent, + CalculatedFieldTestArgumentsComponent, ], imports: [ CommonModule, @@ -338,7 +366,8 @@ import { DashboardViewComponent } from '@home/components/dashboard-view/dashboar SnmpDeviceProfileTransportModule, StatesControllerModule, DeviceCredentialsModule, - DeviceProfileCommonModule + DeviceProfileCommonModule, + EntityDebugSettingsButtonComponent ], exports: [ RouterTabsComponent, @@ -463,11 +492,19 @@ import { DashboardViewComponent } from '@home/components/dashboard-view/dashboar RateLimitsDetailsDialogComponent, SendNotificationButtonComponent, EntityChipsComponent, - DashboardViewComponent + DashboardViewComponent, + CalculatedFieldsTableComponent, + CalculatedFieldDialogComponent, + CalculatedFieldArgumentsTableComponent, + CalculatedFieldArgumentPanelComponent, + CalculatedFieldDebugDialogComponent, + CalculatedFieldScriptTestDialogComponent, + CalculatedFieldTestArgumentsComponent, ], providers: [ WidgetComponentService, CustomDialogService, + DurationLeftPipe, {provide: EMBED_DASHBOARD_DIALOG_TOKEN, useValue: EmbedDashboardDialogComponent}, {provide: COMPLEX_FILTER_PREDICATE_DIALOG_COMPONENT_TOKEN, useValue: ComplexFilterPredicateDialogComponent}, {provide: DASHBOARD_PAGE_COMPONENT_TOKEN, useValue: DashboardPageComponent}, diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html index 9f1839afec..64d5039914 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html @@ -229,6 +229,92 @@ +
    + + {{ 'tenant-profile.calculated-fields' | translate }} tenant-profile.unlimited + +
    + + tenant-profile.max-calculated-fields + + + {{ 'tenant-profile.max-calculated-fields-required' | translate}} + + + {{ 'tenant-profile.max-calculated-fields-range' | translate}} + + + + + tenant-profile.max-data-points-per-rolling-arg + + + {{ 'tenant-profile.max-data-points-per-rolling-arg-required' | translate}} + + + {{ 'tenant-profile.max-data-points-per-rolling-arg-range' | translate}} + + + +
    +
    + + tenant-profile.max-arguments-per-cf + + + {{ 'tenant-profile.max-arguments-per-cf-required' | translate}} + + + {{ 'tenant-profile.max-arguments-per-cf-range' | translate}} + + + +
    +
    + + + + tenant-profile.advanced-settings + + + +
    + + tenant-profile.max-state-size + + + {{ 'tenant-profile.max-state-size-required' | translate}} + + + {{ 'tenant-profile.max-state-size-range' | translate}} + + + + + tenant-profile.max-value-argument-size + + + {{ 'tenant-profile.max-value-argument-size-required' | translate}} + + + {{ 'tenant-profile.max-value-argument-size-range' | translate}} + + + +
    +     +     +
    @@ -638,6 +724,12 @@ [type]="rateLimitsType.EDGE_UPLINK_MESSAGES_PER_EDGE_RATE_LIMIT"> +
    + + +
    +
    diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts index 4cce34c502..b1d6652e4a 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts @@ -118,7 +118,13 @@ export class DefaultTenantProfileConfigurationComponent implements ControlValueA edgeEventRateLimits: [null, []], edgeEventRateLimitsPerEdge: [null, []], edgeUplinkMessagesRateLimits: [null, []], - edgeUplinkMessagesRateLimitsPerEdge: [null, []] + edgeUplinkMessagesRateLimitsPerEdge: [null, []], + maxCalculatedFieldsPerEntity: [null, [Validators.required, Validators.min(0)]], + maxArgumentsPerCF: [null, [Validators.required, Validators.min(0)]], + maxDataPointsPerRollingArg: [null, [Validators.required, Validators.min(0)]], + maxStateSizeInKBytes: [null, [Validators.required, Validators.min(0)]], + calculatedFieldDebugEventsRateLimit: [null, []], + maxSingleValueArgumentSizeInKBytes: [null, [Validators.required, Validators.min(0)]], }); this.defaultTenantProfileConfigurationFormGroup.get('smsEnabled').valueChanges.pipe( diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/rate-limits/rate-limits.models.ts b/ui-ngx/src/app/modules/home/components/profile/tenant/rate-limits/rate-limits.models.ts index ab50c967bf..f09f950ee7 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/rate-limits/rate-limits.models.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/rate-limits/rate-limits.models.ts @@ -45,7 +45,8 @@ export enum RateLimitsType { EDGE_EVENTS_RATE_LIMIT = 'EDGE_EVENTS_RATE_LIMIT', EDGE_EVENTS_PER_EDGE_RATE_LIMIT = 'EDGE_EVENTS_PER_EDGE_RATE_LIMIT', EDGE_UPLINK_MESSAGES_RATE_LIMIT = 'EDGE_UPLINK_MESSAGES_RATE_LIMIT', - EDGE_UPLINK_MESSAGES_PER_EDGE_RATE_LIMIT = 'EDGE_UPLINK_MESSAGES_PER_EDGE_RATE_LIMIT' + EDGE_UPLINK_MESSAGES_PER_EDGE_RATE_LIMIT = 'EDGE_UPLINK_MESSAGES_PER_EDGE_RATE_LIMIT', + CALCULATED_FIELD_DEBUG_EVENT_RATE_LIMIT = 'CALCULATED_FIELD_DEBUG_EVENT_RATE_LIMIT', } export const rateLimitsLabelTranslationMap = new Map( @@ -74,6 +75,7 @@ export const rateLimitsLabelTranslationMap = new Map( [RateLimitsType.EDGE_EVENTS_PER_EDGE_RATE_LIMIT, 'tenant-profile.rate-limits.edge-events-per-edge-rate-limit'], [RateLimitsType.EDGE_UPLINK_MESSAGES_RATE_LIMIT, 'tenant-profile.rate-limits.edge-uplink-messages-rate-limit'], [RateLimitsType.EDGE_UPLINK_MESSAGES_PER_EDGE_RATE_LIMIT, 'tenant-profile.rate-limits.edge-uplink-messages-per-edge-rate-limit'], + [RateLimitsType.CALCULATED_FIELD_DEBUG_EVENT_RATE_LIMIT, 'tenant-profile.rate-limits.calculated-field-debug-event-rate-limit'], ] ); @@ -103,6 +105,7 @@ export const rateLimitsDialogTitleTranslationMap = new Map -
    +
    {{ title }}
    rule-node-config.save-time-series.strategy - @for (strategy of persistenceStrategies; track strategy) { - {{ PersistenceTypeTranslationMap.get(strategy) | translate }} + @for (strategy of processingStrategies; track strategy) { + {{ ProcessingTypeTranslationMap.get(strategy) | translate }} } - @if(persistenceSettingRowForm.get('type').value === PersistenceType.DEDUPLICATE) { + @if (processingSettingRowForm.get('type').value === ProcessingType.DEDUPLICATE) { AdvancedPersistenceSettingRowComponent), + useExisting: forwardRef(() => AdvancedProcessingSettingRowComponent), multi: true },{ provide: NG_VALIDATORS, - useExisting: forwardRef(() => AdvancedPersistenceSettingRowComponent), + useExisting: forwardRef(() => AdvancedProcessingSettingRowComponent), multi: true }] }) -export class AdvancedPersistenceSettingRowComponent implements ControlValueAccessor, Validator { +export class AdvancedProcessingSettingRowComponent implements ControlValueAccessor, Validator { @Input() title: string; - persistenceSettingRowForm = this.fb.group({ + processingSettingRowForm = this.fb.group({ type: [defaultAdvancedProcessingConfig.type], deduplicationIntervalSecs: [{value: 60, disabled: true}] }); - PersistenceType = ProcessingType; - persistenceStrategies = [ProcessingType.ON_EVERY_MESSAGE, ProcessingType.DEDUPLICATE, ProcessingType.SKIP]; - PersistenceTypeTranslationMap = ProcessingTypeTranslationMap; + ProcessingType = ProcessingType; + processingStrategies = [ProcessingType.ON_EVERY_MESSAGE, ProcessingType.DEDUPLICATE, ProcessingType.SKIP]; + ProcessingTypeTranslationMap = ProcessingTypeTranslationMap; maxDeduplicateTime = maxDeduplicateTimeSecs; private propagateChange: (value: any) => void = () => {}; constructor(private fb: FormBuilder) { - this.persistenceSettingRowForm.get('type').valueChanges.pipe( + this.processingSettingRowForm.get('type').valueChanges.pipe( takeUntilDestroyed() ).subscribe(() => this.updatedValidation()); - this.persistenceSettingRowForm.valueChanges.pipe( + this.processingSettingRowForm.valueChanges.pipe( takeUntilDestroyed() ).subscribe((value) => this.propagateChange(value)); } @@ -83,32 +83,32 @@ export class AdvancedPersistenceSettingRowComponent implements ControlValueAcces setDisabledState(isDisabled: boolean) { if (isDisabled) { - this.persistenceSettingRowForm.disable({emitEvent: false}); + this.processingSettingRowForm.disable({emitEvent: false}); } else { - this.persistenceSettingRowForm.enable({emitEvent: false}); + this.processingSettingRowForm.enable({emitEvent: false}); this.updatedValidation(); } } validate(): ValidationErrors | null { - return this.persistenceSettingRowForm.valid ? null : { - persistenceSettingRow: false + return this.processingSettingRowForm.valid ? null : { + processingSettingRow: false }; } writeValue(value: AdvancedProcessingConfig) { if (isDefinedAndNotNull(value)) { - this.persistenceSettingRowForm.patchValue(value, {emitEvent: false}); + this.processingSettingRowForm.patchValue(value, {emitEvent: false}); } else { - this.persistenceSettingRowForm.patchValue(defaultAdvancedProcessingConfig); + this.processingSettingRowForm.patchValue(defaultAdvancedProcessingConfig); } } private updatedValidation() { - if (this.persistenceSettingRowForm.get('type').value === ProcessingType.DEDUPLICATE) { - this.persistenceSettingRowForm.get('deduplicationIntervalSecs').enable({emitEvent: false}); + if (this.processingSettingRowForm.get('type').value === ProcessingType.DEDUPLICATE) { + this.processingSettingRowForm.get('deduplicationIntervalSecs').enable({emitEvent: false}); } else { - this.persistenceSettingRowForm.get('deduplicationIntervalSecs').disable({emitEvent: false}) + this.processingSettingRowForm.get('deduplicationIntervalSecs').disable({emitEvent: false}) } } } diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-processing-setting.component.html similarity index 68% rename from ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.html rename to ui-ngx/src/app/modules/home/components/rule-node/action/advanced-processing-setting.component.html index 094f6dbc2a..d72e7f1e43 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-processing-setting.component.html @@ -15,22 +15,26 @@ limitations under the License. --> -
    +
    - - + - + + > +
    diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-processing-setting.component.ts similarity index 69% rename from ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.ts rename to ui-ngx/src/app/modules/home/components/rule-node/action/advanced-processing-setting.component.ts index 01314ec4f3..9e204c0219 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-processing-setting.component.ts @@ -27,30 +27,31 @@ import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { AdvancedProcessingStrategy } from '@home/components/rule-node/action/timeseries-config.models'; @Component({ - selector: 'tb-advanced-persistence-settings', - templateUrl: './advanced-persistence-setting.component.html', + selector: 'tb-advanced-processing-settings', + templateUrl: './advanced-processing-setting.component.html', providers: [{ provide: NG_VALUE_ACCESSOR, - useExisting: forwardRef(() => AdvancedPersistenceSettingComponent), + useExisting: forwardRef(() => AdvancedProcessingSettingComponent), multi: true },{ provide: NG_VALIDATORS, - useExisting: forwardRef(() => AdvancedPersistenceSettingComponent), + useExisting: forwardRef(() => AdvancedProcessingSettingComponent), multi: true }] }) -export class AdvancedPersistenceSettingComponent implements ControlValueAccessor, Validator { +export class AdvancedProcessingSettingComponent implements ControlValueAccessor, Validator { - persistenceForm = this.fb.group({ + processingForm = this.fb.group({ timeseries: [null], latest: [null], - webSockets: [null] + webSockets: [null], + calculatedFields: [null] }); private propagateChange: (value: any) => void = () => {}; constructor(private fb: FormBuilder) { - this.persistenceForm.valueChanges.pipe( + this.processingForm.valueChanges.pipe( takeUntilDestroyed() ).subscribe(value => this.propagateChange(value)); } @@ -64,19 +65,19 @@ export class AdvancedPersistenceSettingComponent implements ControlValueAccessor setDisabledState(isDisabled: boolean) { if (isDisabled) { - this.persistenceForm.disable({emitEvent: false}); + this.processingForm.disable({emitEvent: false}); } else { - this.persistenceForm.enable({emitEvent: false}); + this.processingForm.enable({emitEvent: false}); } } validate(): ValidationErrors | null { - return this.persistenceForm.valid ? null : { - persistenceForm: false + return this.processingForm.valid ? null : { + processingForm: false }; } writeValue(value: AdvancedProcessingStrategy) { - this.persistenceForm.patchValue(value, {emitEvent: false}); + this.processingForm.patchValue(value, {emitEvent: false}); } } diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html index 6e3e9fb6d1..d7311d2d51 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html @@ -31,13 +31,13 @@ rule-node-config.save-time-series.strategy - @for (strategy of persistenceStrategies; track strategy) { - {{ PersistenceTypeTranslationMap.get(strategy) | translate }} + @for (strategy of processingStrategies; track strategy) { + {{ ProcessingTypeTranslationMap.get(strategy) | translate }} } - @if(timeseriesConfigForm.get('processingSettings.type').value === PersistenceType.DEDUPLICATE) { + @if(timeseriesConfigForm.get('processingSettings.type').value === ProcessingType.DEDUPLICATE) { } } @else { - + > }
    diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts index 47c0501b26..7f27bca241 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts @@ -18,7 +18,7 @@ import { Component } from '@angular/core'; import { FormBuilder, FormGroup, Validators } from '@angular/forms'; import { RuleNodeConfigurationComponent } from '@shared/models/rule-node.models'; import { - defaultAdvancedPersistenceStrategy, + defaultAdvancedProcessingStrategy, maxDeduplicateTimeSecs, ProcessingSettings, ProcessingSettingsForm, @@ -37,9 +37,9 @@ export class TimeseriesConfigComponent extends RuleNodeConfigurationComponent { timeseriesConfigForm: FormGroup; - PersistenceType = ProcessingType; - persistenceStrategies = [ProcessingType.ON_EVERY_MESSAGE, ProcessingType.DEDUPLICATE, ProcessingType.WEBSOCKETS_ONLY]; - PersistenceTypeTranslationMap = ProcessingTypeTranslationMap; + ProcessingType = ProcessingType; + processingStrategies = [ProcessingType.ON_EVERY_MESSAGE, ProcessingType.DEDUPLICATE, ProcessingType.WEBSOCKETS_ONLY]; + ProcessingTypeTranslationMap = ProcessingTypeTranslationMap; maxDeduplicateTime = maxDeduplicateTimeSecs @@ -63,14 +63,14 @@ export class TimeseriesConfigComponent extends RuleNodeConfigurationComponent { type: isAdvanced ? ProcessingType.ON_EVERY_MESSAGE : config.processingSettings.type, isAdvanced: isAdvanced, deduplicationIntervalSecs: config.processingSettings?.deduplicationIntervalSecs ?? 60, - advanced: isAdvanced ? config.processingSettings : defaultAdvancedPersistenceStrategy + advanced: isAdvanced ? config.processingSettings : defaultAdvancedProcessingStrategy } } else { processingSettings = { type: ProcessingType.ON_EVERY_MESSAGE, isAdvanced: false, deduplicationIntervalSecs: 60, - advanced: defaultAdvancedPersistenceStrategy + advanced: defaultAdvancedProcessingStrategy }; } return { diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts index f8987b7f0e..9785125999 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts @@ -55,14 +55,15 @@ export interface BasicProcessingSettings { type: ProcessingType; } -export interface DeduplicateProcessingStrategy extends BasicProcessingSettings{ +export interface DeduplicateProcessingStrategy extends BasicProcessingSettings { deduplicationIntervalSecs: number; } -export interface AdvancedProcessingStrategy extends BasicProcessingSettings{ +export interface AdvancedProcessingStrategy extends BasicProcessingSettings { timeseries: AdvancedProcessingConfig; latest: AdvancedProcessingConfig; webSockets: AdvancedProcessingConfig; + calculatedFields: AdvancedProcessingConfig; } export type AdvancedProcessingConfig = WithOptional; @@ -71,8 +72,9 @@ export const defaultAdvancedProcessingConfig: AdvancedProcessingConfig = { type: ProcessingType.ON_EVERY_MESSAGE } -export const defaultAdvancedPersistenceStrategy: Omit = { +export const defaultAdvancedProcessingStrategy: Omit = { timeseries: defaultAdvancedProcessingConfig, latest: defaultAdvancedProcessingConfig, webSockets: defaultAdvancedProcessingConfig, + calculatedFields: defaultAdvancedProcessingConfig, } diff --git a/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts b/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts index d2435c8641..5117aa6430 100644 --- a/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts +++ b/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts @@ -121,7 +121,11 @@ export class EntityLinkTableColumn> extends BaseEntity public width: string = '0px', public cellContentFunction: CellContentFunction = (entity, property) => entity[property] ? entity[property] : '', public entityURL: (entity) => string, - public sortable: boolean = true) { + public sortable: boolean = true, + public cellStyleFunction: CellStyleFunction = () => ({}), + public headerCellStyleFunction: HeaderCellStyleFunction = () => ({}), + public cellTooltipFunction: CellTooltipFunction = () => undefined, + public actionCell: CellActionDescriptor = null) { super('link', key, title, width, sortable); } } diff --git a/ui-ngx/src/app/modules/home/models/entity/entity-table-component.models.ts b/ui-ngx/src/app/modules/home/models/entity/entity-table-component.models.ts index 3accf6bfb9..6317f267ca 100644 --- a/ui-ngx/src/app/modules/home/models/entity/entity-table-component.models.ts +++ b/ui-ngx/src/app/modules/home/models/entity/entity-table-component.models.ts @@ -20,7 +20,7 @@ import { SafeHtml } from '@angular/platform-browser'; import { PageLink } from '@shared/models/page/page-link'; import { Timewindow } from '@shared/models/time/time.models'; import { EntitiesDataSource } from '@home/models/datasource/entity-datasource'; -import { ElementRef, EventEmitter } from '@angular/core'; +import { ElementRef, EventEmitter, Renderer2, ViewContainerRef } from '@angular/core'; import { TbAnchorComponent } from '@shared/components/tb-anchor.component'; import { MatPaginator } from '@angular/material/paginator'; import { MatSort } from '@angular/material/sort'; @@ -64,6 +64,7 @@ export interface IEntitiesTableComponent { paginator: MatPaginator; sort: MatSort; route: ActivatedRoute; + viewContainerRef: ViewContainerRef; addEnabled(): boolean; clearSelection(): void; diff --git a/ui-ngx/src/app/modules/home/pages/admin/resource/js-library-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/admin/resource/js-library-table-config.resolver.ts index 314ae7809f..9c5dba6dbb 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/resource/js-library-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/resource/js-library-table-config.resolver.ts @@ -16,6 +16,7 @@ import { Injectable } from '@angular/core'; import { + CellActionDescriptor, checkBoxCell, DateEntityTableColumn, EntityTableColumn, @@ -27,7 +28,9 @@ import { ResourceInfo, ResourceSubType, ResourceSubTypeTranslationMap, - ResourceType + ResourceType, + ResourceInfoWithReferences, + toResourceDeleteResult } from '@shared/models/resource.models'; import { EntityType, entityTypeResources } from '@shared/models/entity-type.models'; import { NULL_UUID } from '@shared/models/id/has-uuid'; @@ -42,8 +45,19 @@ import { PageLink } from '@shared/models/page/page-link'; import { EntityAction } from '@home/models/entity/entity-component.models'; import { JsLibraryTableHeaderComponent } from '@home/pages/admin/resource/js-library-table-header.component'; import { JsResourceComponent } from '@home/pages/admin/resource/js-resource.component'; -import { switchMap } from 'rxjs/operators'; +import { catchError, map, switchMap } from 'rxjs/operators'; import { ResourceTabsComponent } from '@home/pages/admin/resource/resource-tabs.component'; +import { forkJoin, of } from 'rxjs'; +import { parseHttpErrorMessage } from '@core/utils'; +import { ActionNotificationShow } from '@core/notification/notification.actions'; +import { MatDialog } from '@angular/material/dialog'; +import { DialogService } from '@core/services/dialog.service'; +import { + ResourcesInUseDialogComponent, + ResourcesInUseDialogData +} from "@shared/components/resource/resources-in-use-dialog.component"; +import { ResourcesDatasource } from "@home/pages/admin/resource/resources-datasource"; +import { AuthUser } from '@shared/models/user.model'; @Injectable() export class JsLibraryTableConfigResolver { @@ -53,6 +67,8 @@ export class JsLibraryTableConfigResolver { constructor(private store: Store, private resourceService: ResourceService, private translate: TranslateService, + private dialog: MatDialog, + private dialogService: DialogService, private router: Router, private datePipe: DatePipe) { @@ -81,20 +97,16 @@ export class JsLibraryTableConfigResolver { entity => checkBoxCell(entity.tenantId.id === NULL_UUID)), ); - this.config.cellActionDescriptors.push( - { - name: this.translate.instant('javascript.download'), - icon: 'file_download', - isEnabled: () => true, - onAction: ($event, entity) => this.downloadResource($event, entity) - } - ); + this.config.cellActionDescriptors = this.configureCellActions(getCurrentAuthUser(this.store)); - this.config.deleteEntityTitle = resource => this.translate.instant('javascript.delete-javascript-resource-title', - { resourceTitle: resource.title }); - this.config.deleteEntityContent = () => this.translate.instant('javascript.delete-javascript-resource-text'); - this.config.deleteEntitiesTitle = count => this.translate.instant('javascript.delete-javascript-resources-title', {count}); - this.config.deleteEntitiesContent = () => this.translate.instant('javascript.delete-javascript-resources-text'); + this.config.groupActionDescriptors = [{ + name: this.translate.instant('action.delete'), + icon: 'delete', + isEnabled: true, + onAction: ($event, entities) => this.deleteResources($event, entities) + }]; + + this.config.entitiesDeleteEnabled = false; this.config.entitiesFetchFunction = pageLink => this.resourceService.getResources(pageLink, ResourceType.JS_MODULE, this.config.componentsData.resourceSubType); this.config.loadEntity = id => { @@ -115,7 +127,6 @@ export class JsLibraryTableConfigResolver { } return saveObservable; }; - this.config.deleteEntity = id => this.resourceService.deleteResource(id.id); this.config.onEntityAction = action => this.onResourceAction(action); } @@ -126,7 +137,6 @@ export class JsLibraryTableConfigResolver { resourceSubType: '' }; const authUser = getCurrentAuthUser(this.store); - this.config.deleteEnabled = (resource) => this.isResourceEditable(resource, authUser.authority); this.config.entitySelectionEnabled = (resource) => this.isResourceEditable(resource, authUser.authority); this.config.detailsReadonly = (resource) => this.detailsReadonly(resource, authUser.authority); return this.config; @@ -170,4 +180,151 @@ export class JsLibraryTableConfigResolver { return authority === Authority.SYS_ADMIN; } } + + private deleteResource($event: Event, resource: ResourceInfo) { + if ($event) { + $event.stopPropagation(); + } + this.dialogService.confirm( + this.translate.instant('javascript.delete-javascript-resource-title', { resourceTitle: resource.title }), + this.translate.instant('javascript.delete-javascript-resource-text'), + this.translate.instant('action.no'), + this.translate.instant('action.yes'), + true + ).subscribe((result) => { + if (result) { + this.resourceService.deleteResource(resource.id.id, false, {ignoreErrors: true}).pipe( + map(() => toResourceDeleteResult(resource)), + catchError((err) => of(toResourceDeleteResult(resource, err))) + ).subscribe( + (deleteResult) => { + if (deleteResult.success) { + this.config.updateData(); + } else if (deleteResult.resourceIsReferencedError) { + const resources: ResourceInfoWithReferences[] = [{...resource, ...{references: deleteResult.references}}]; + const data = { + multiple: false, + resources, + configuration: { + title: 'javascript.javascript-resource-is-in-use', + message: this.translate.instant('javascript.javascript-resource-is-in-use-text', {title: resources[0].title}), + deleteText: 'javascript.delete-javascript-resource-in-use-text', + selectedText: 'javascript.selected-javascript-resources', + columns: ['select', 'title', 'references'] + } + }; + this.dialog.open(ResourcesInUseDialogComponent, { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], + data + }).afterClosed().subscribe((resources) => { + if (resources) { + this.resourceService.deleteResource(resource.id.id, true).subscribe( + () => { + this.config.updateData(); + } + ); + } + }); + } else { + const errorMessageWithTimeout = parseHttpErrorMessage(deleteResult.error, this.translate); + setTimeout(() => { + this.store.dispatch(new ActionNotificationShow({message: errorMessageWithTimeout.message, type: 'error'})); + }, errorMessageWithTimeout.timeout); + } + } + ); + } + }); + } + + private deleteResources($event: Event, resources: ResourceInfo[]) { + if ($event) { + $event.stopPropagation(); + } + if (resources && resources.length) { + const title = this.translate.instant('javascript.delete-javascript-resources-title', {count: resources.length}); + const content = this.translate.instant('javascript.delete-javascript-resources-text'); + this.dialogService.confirm(title, content, + this.translate.instant('action.no'), + this.translate.instant('action.yes')).subscribe((result) => { + if (result) { + const tasks = resources.map((resource) => + this.resourceService.deleteResource(resource.id.id, false, {ignoreErrors: true}).pipe( + map(() => toResourceDeleteResult(resource)), + catchError((err) => of(toResourceDeleteResult(resource, err))) + ) + ); + forkJoin(tasks).subscribe( + (deleteResults) => { + const anySuccess = deleteResults.some(res => res.success); + const referenceErrors = deleteResults.filter(res => res.resourceIsReferencedError); + const otherError = deleteResults.find(res => !res.success); + if (anySuccess) { + this.config.updateData(); + } + if (referenceErrors?.length) { + const resourcesWithReferences: ResourceInfoWithReferences[] = + referenceErrors.map(ref => ({...ref.resource, ...{references: ref.references}})); + const data = { + multiple: true, + resources: resourcesWithReferences, + configuration: { + title: 'javascript.javascript-resources-are-in-use', + message: this.translate.instant('javascript.javascript-resources-are-in-use-text'), + deleteText: 'javascript.delete-javascript-resource-in-use-text', + selectedText: 'javascript.selected-javascript-resources', + datasource: new ResourcesDatasource(this.resourceService, resourcesWithReferences, entity => true), + columns: ['select', 'title', 'references'] + } + }; + this.dialog.open(ResourcesInUseDialogComponent, { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], + data + }).afterClosed().subscribe((forceDeleteResources) => { + if (forceDeleteResources && forceDeleteResources.length) { + const forceDeleteTasks = forceDeleteResources.map((resource) => + this.resourceService.deleteResource(resource.id.id, true) + ); + forkJoin(forceDeleteTasks).subscribe( + () => { + this.config.updateData(); + } + ); + } + }); + } else if (otherError) { + const errorMessageWithTimeout = parseHttpErrorMessage(otherError.error, this.translate); + setTimeout(() => { + this.store.dispatch(new ActionNotificationShow({message: errorMessageWithTimeout.message, type: 'error'})); + }, errorMessageWithTimeout.timeout); + } + } + ); + } + }); + } + } + + private configureCellActions(authUser: AuthUser): Array> { + const actions: Array> = []; + actions.push( + { + name: this.translate.instant('javascript.download'), + icon: 'file_download', + isEnabled: () => true, + onAction: ($event, entity) => this.downloadResource($event, entity) + }, + { + name: this.translate.instant('javascript.delete'), + icon: 'delete', + isEnabled: (resource) => this.isResourceEditable(resource, authUser.authority), + onAction: ($event, entity) => this.deleteResource($event, entity) + }, + ); + return actions; + } } diff --git a/ui-ngx/src/app/modules/home/pages/admin/resource/resources-datasource.ts b/ui-ngx/src/app/modules/home/pages/admin/resource/resources-datasource.ts new file mode 100644 index 0000000000..ed9c9f5dd9 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/resource/resources-datasource.ts @@ -0,0 +1,130 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { CollectionViewer, DataSource, SelectionModel } from '@angular/cdk/collections'; +import { ResourceInfo, ResourceSubType, ResourceType } from '@shared/models/resource.models'; +import { BehaviorSubject, Observable, of, ReplaySubject, Subject } from 'rxjs'; +import { emptyPageData, PageData } from '@shared/models/page/page-data'; +import { EntityBooleanFunction } from '@home/models/entity/entities-table-config.models'; +import { PageLink } from '@shared/models/page/page-link'; +import { catchError, map, take, tap } from 'rxjs/operators'; +import { ResourceService } from "@core/http/resource.service"; + +export class ResourcesDatasource implements DataSource { + private entitiesSubject: Subject; + private readonly pageDataSubject: Subject>; + + public pageData$: Observable>; + + public selection = new SelectionModel(true, []); + + public dataLoading = true; + + constructor(private resourceService: ResourceService, + private resources: ResourceInfo[], + private selectionEnabledFunction: EntityBooleanFunction) { + if (this.resources && this.resources.length) { + this.entitiesSubject = new BehaviorSubject(this.resources); + } else { + this.entitiesSubject = new BehaviorSubject([]); + this.pageDataSubject = new BehaviorSubject>(emptyPageData()); + this.pageData$ = this.pageDataSubject.asObservable(); + } + } + + connect(collectionViewer: CollectionViewer): + Observable> { + return this.entitiesSubject.asObservable(); + } + + disconnect(collectionViewer: CollectionViewer): void { + this.entitiesSubject.complete(); + if (this.pageDataSubject) { + this.pageDataSubject.complete(); + } + } + + reset() { + this.entitiesSubject.next([]); + if (this.pageDataSubject) { + this.pageDataSubject.next(emptyPageData()); + } + } + + loadEntities(pageLink: PageLink, resourceType: ResourceType, subType: ResourceSubType): Observable> { + this.dataLoading = true; + const result = new ReplaySubject>(); + this.fetchEntities(pageLink, resourceType, subType).pipe( + tap(() => { + this.selection.clear(); + }), + catchError(() => of(emptyPageData())), + ).subscribe( + (pageData) => { + this.entitiesSubject.next(pageData.data); + this.pageDataSubject.next(pageData); + result.next(pageData); + this.dataLoading = false; + } + ); + return result; + } + + fetchEntities(pageLink: PageLink, resourceType: ResourceType, subType: ResourceSubType): Observable> { + return this.resourceService.getResources(pageLink, resourceType, subType); + } + + isAllSelected(): Observable { + const numSelected = this.selection.selected.length; + return this.entitiesSubject.pipe( + map((entities) => numSelected === entities.length) + ); + } + + isEmpty(): Observable { + return this.entitiesSubject.pipe( + map((entities) => !entities.length) + ); + } + + total(): Observable { + return this.pageDataSubject.pipe( + map((pageData) => pageData.totalElements) + ); + } + + masterToggle() { + this.entitiesSubject.pipe( + tap((entities) => { + const numSelected = this.selection.selected.length; + if (numSelected === this.selectableEntitiesCount(entities)) { + this.selection.clear(); + } else { + entities.forEach(row => { + if (this.selectionEnabledFunction(row)) { + this.selection.select(row); + } + }); + } + }), + take(1) + ).subscribe(); + } + + private selectableEntitiesCount(entities: Array): number { + return entities.filter((entity) => this.selectionEnabledFunction(entity)).length; + } +} diff --git a/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html b/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html index ea93ea4c6b..e4431abfed 100644 --- a/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html @@ -15,6 +15,10 @@ limitations under the License. --> + + + diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html index 0cd3e759b9..767d11eb23 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html @@ -32,6 +32,10 @@ [entityName]="entity.name"> + + + diff --git a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html index c45491a608..46aa08ff9f 100644 --- a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html @@ -40,6 +40,10 @@ + + + + + + diff --git a/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol-editor.models.ts b/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol-editor.models.ts index f6d37d3390..13b55d4f67 100644 --- a/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol-editor.models.ts +++ b/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol-editor.models.ts @@ -31,7 +31,12 @@ import { } from '@home/components/widget/lib/scada/scada-symbol.models'; import { TbEditorCompletion, TbEditorCompletions } from '@shared/models/ace/completion.models'; import { CustomTranslatePipe } from '@shared/pipe/custom-translate.pipe'; -import { AceHighlightRule, AceHighlightRules } from '@shared/models/ace/ace.models'; +import { + AceHighlightRule, + AceHighlightRules, + dotOperatorHighlightRule, + endGroupHighlightRule +} from '@shared/models/ace/ace.models'; import { HelpLinks, ValueType } from '@shared/models/constants'; import { formPropertyCompletions } from '@shared/models/dynamic-form.models'; import ITooltipsterInstance = JQueryTooltipster.ITooltipsterInstance; @@ -921,17 +926,6 @@ export class ScadaSymbolElement { const identifierRe = /[a-zA-Z$_\u00a1-\uffff][a-zA-Z\d$_\u00a1-\uffff]*/; -const dotOperatorHighlightRule: AceHighlightRule = { - token: 'punctuation.operator', - regex: /[.](?![.])/, -}; - -const endGroupHighlightRule: AceHighlightRule = { - regex: '', - token: 'empty', - next: 'no_regex' -}; - const scadaSymbolCtxObjectHighlightRule: AceHighlightRule = { token: 'tb.scada-symbol-ctx', regex: /\bctx\b/, diff --git a/ui-ngx/src/app/shared/components/dialog/json-object-edit-dialog.component.html b/ui-ngx/src/app/shared/components/dialog/json-object-edit-dialog.component.html index 15a462e7f6..5047302bc7 100644 --- a/ui-ngx/src/app/shared/components/dialog/json-object-edit-dialog.component.html +++ b/ui-ngx/src/app/shared/components/dialog/json-object-edit-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
    +

    {{ title }}

    @@ -25,30 +25,26 @@ close     - - -
    -
    -
    - - -
    +
    +
    + +
    diff --git a/ui-ngx/src/app/shared/components/dialog/json-object-edit-dialog.component.ts b/ui-ngx/src/app/shared/components/dialog/json-object-edit-dialog.component.ts index d409e4a80c..d6f733ade7 100644 --- a/ui-ngx/src/app/shared/components/dialog/json-object-edit-dialog.component.ts +++ b/ui-ngx/src/app/shared/components/dialog/json-object-edit-dialog.component.ts @@ -30,6 +30,7 @@ export interface JsonObjectEditDialogData { title?: string; saveLabel?: string; cancelLabel?: string; + fillHeight?: boolean; } @Component({ diff --git a/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html b/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html index ca1fbe412a..af63bae6a7 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html +++ b/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html @@ -15,12 +15,17 @@ limitations under the License. --> - - {{ label | translate }} + {{ label | translate }} {{ displayEntityFn(selectEntityFormGroup.get('entity').value) }} + + warning + + } @else if (keyControl.hasError('required') && keyControl.touched) { + + warning + + } + + @for (key of filteredKeys$ | async; track key) { + + } @empty { + @if (!this.keyControl.value) { + {{ 'entity.no-keys-found' | translate }} + } + } + + diff --git a/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts b/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts new file mode 100644 index 0000000000..84d723d114 --- /dev/null +++ b/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts @@ -0,0 +1,145 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, effect, ElementRef, forwardRef, input, OnChanges, SimpleChanges, ViewChild, } from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; +import { map, startWith, switchMap } from 'rxjs/operators'; +import { combineLatest, of, Subject } from 'rxjs'; +import { EntityService } from '@core/http/entity.service'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { AttributeScope, DataKeyType } from '@shared/models/telemetry/telemetry.models'; +import { EntitiesKeysByQuery } from '@shared/models/entity.models'; +import { EntityFilter } from '@shared/models/query/query.models'; +import { isEqual } from '@core/utils'; + +@Component({ + selector: 'tb-entity-key-autocomplete', + templateUrl: './entity-key-autocomplete.component.html', + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => EntityKeyAutocompleteComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => EntityKeyAutocompleteComponent), + multi: true + } + ], +}) +export class EntityKeyAutocompleteComponent implements ControlValueAccessor, Validator, OnChanges { + + @ViewChild('keyInput', {static: true}) keyInput: ElementRef; + + entityFilter = input.required(); + dataKeyType = input.required(); + keyScopeType = input(); + + keyControl = this.fb.control('', [Validators.required]); + searchText = ''; + keyInputSubject = new Subject(); + + private propagateChange: (value: string) => void; + private cachedResult: EntitiesKeysByQuery; + + keys$ = this.keyInputSubject.asObservable() + .pipe( + switchMap(() => { + return this.cachedResult ? of(this.cachedResult) : this.entityService.findEntityKeysByQuery({ + pageLink: { page: 0, pageSize: 100 }, + entityFilter: this.entityFilter(), + }, this.dataKeyType() === DataKeyType.attribute, this.dataKeyType() === DataKeyType.timeseries, this.keyScopeType()); + }), + map(result => { + this.cachedResult = result; + switch (this.dataKeyType()) { + case DataKeyType.attribute: + return result.attribute; + case DataKeyType.timeseries: + return result.timeseries; + default: + return []; + } + }), + ); + + filteredKeys$ = combineLatest([this.keys$, this.keyControl.valueChanges.pipe(startWith(''))]) + .pipe( + map(([keys, searchText = '']) => { + this.searchText = searchText; + return searchText ? keys.filter(item => item.toLowerCase().includes(searchText.toLowerCase())) : keys; + }) + ); + + constructor( + private fb: FormBuilder, + private entityService: EntityService, + ) { + this.keyControl.valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(value => this.propagateChange(value)); + effect(() => { + if (this.keyScopeType() || this.entityFilter() && this.dataKeyType()) { + this.cachedResult = null; + this.searchText = ''; + } + }); + } + + ngOnChanges(changes: SimpleChanges): void { + const filterChanged = changes.entityFilter?.previousValue && + !isEqual(changes.entityFilter.currentValue, changes.entityFilter.previousValue); + const keyScopeChanged = changes.keyScopeType?.previousValue && + changes.keyScopeType.currentValue !== changes.keyScopeType.previousValue; + const keyTypeChanged = changes.dataKeyType?.previousValue && + changes.dataKeyType.currentValue !== changes.dataKeyType.previousValue; + + if (filterChanged || keyScopeChanged || keyTypeChanged) { + this.keyControl.setValue('', {emitEvent: false}); + } + } + + clear(): void { + this.keyControl.patchValue('', {emitEvent: true}); + setTimeout(() => { + this.keyInput.nativeElement.blur(); + this.keyInput.nativeElement.focus(); + }, 0); + } + + registerOnChange(onChange: (value: string) => void): void { + this.propagateChange = onChange; + } + + registerOnTouched(_): void {} + + validate(): ValidationErrors | null { + return this.keyControl.valid ? null : { keyControl: false }; + } + + writeValue(value: string): void { + this.keyControl.patchValue(value, {emitEvent: false}); + } +} diff --git a/ui-ngx/src/app/shared/components/image/image-gallery.component.ts b/ui-ngx/src/app/shared/components/image/image-gallery.component.ts index ef22c037f2..6e40683c2b 100644 --- a/ui-ngx/src/app/shared/components/image/image-gallery.component.ts +++ b/ui-ngx/src/app/shared/components/image/image-gallery.component.ts @@ -16,10 +16,10 @@ import { ImageResourceInfo, - ImageResourceInfoWithReferences, + ResourceInfoWithReferences, imageResourceType, ResourceSubType, - toImageDeleteResult + toResourceDeleteResult } from '@shared/models/resource.models'; import { forkJoin, merge, Observable, of, Subject, Subscription } from 'rxjs'; import { ImageService } from '@core/http/image.service'; @@ -67,9 +67,9 @@ import { ImageDialogComponent, ImageDialogData } from '@shared/components/image/ import { ImportExportService } from '@shared/import-export/import-export.service'; import { ActionNotificationShow } from '@core/notification/notification.actions'; import { - ImagesInUseDialogComponent, - ImagesInUseDialogData -} from '@shared/components/image/images-in-use-dialog.component'; + ResourcesInUseDialogComponent, + ResourcesInUseDialogData +} from '@shared/components/resource/resources-in-use-dialog.component'; import { ImagesDatasource } from '@shared/components/image/images-datasource'; import { EmbedImageDialogComponent, EmbedImageDialogData } from '@shared/components/image/embed-image-dialog.component'; @@ -504,21 +504,30 @@ export class ImageGalleryComponent extends PageComponent implements OnInit, OnDe this.translate.instant('action.yes')).subscribe((result) => { if (result) { this.imageService.deleteImage(imageResourceType(image), image.resourceKey, false, {ignoreErrors: true}).pipe( - map(() => toImageDeleteResult(image)), - catchError((err) => of(toImageDeleteResult(image, err))) + map(() => toResourceDeleteResult(image)), + catchError((err) => of(toResourceDeleteResult(image, err))) ).subscribe( (deleteResult) => { if (deleteResult.success) { this.imageDeleted(itemIndex); - } else if (deleteResult.imageIsReferencedError) { - this.dialog.open(ImagesInUseDialogComponent, { + } else if (deleteResult.resourceIsReferencedError) { + const images = [{...image, ...{references: deleteResult.references}}]; + const data = { + multiple: false, + resources: images, + configuration: { + title: 'image.image-is-in-use', + message: this.translate.instant('image.image-is-in-use-text', {title: images[0].title}), + deleteText: 'image.delete-image-in-use-text', + selectedText: 'image.selected-images', + columns: ['select', 'preview', 'title', 'references'] + } + }; + this.dialog.open(ResourcesInUseDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], - data: { - multiple: false, - images: [{...image, ...{references: deleteResult.references}}] - } + data }).afterClosed().subscribe((images) => { if (images) { this.imageService.deleteImage(imageResourceType(image), image.resourceKey, true).subscribe( @@ -554,29 +563,38 @@ export class ImageGalleryComponent extends PageComponent implements OnInit, OnDe if (result) { const tasks = selectedImages.map((image) => this.imageService.deleteImage(imageResourceType(image), image.resourceKey, false, {ignoreErrors: true}).pipe( - map(() => toImageDeleteResult(image)), - catchError((err) => of(toImageDeleteResult(image, err))) + map(() => toResourceDeleteResult(image)), + catchError((err) => of(toResourceDeleteResult(image, err))) ) ); forkJoin(tasks).subscribe( (deleteResults) => { const anySuccess = deleteResults.some(res => res.success); - const referenceErrors = deleteResults.filter(res => res.imageIsReferencedError); + const referenceErrors = deleteResults.filter(res => res.resourceIsReferencedError); const otherError = deleteResults.find(res => !res.success); if (anySuccess) { this.updateData(); } if (referenceErrors?.length) { - const imagesWithReferences: ImageResourceInfoWithReferences[] = - referenceErrors.map(ref => ({...ref.image, ...{references: ref.references}})); - this.dialog.open(ImagesInUseDialogComponent, { + const imagesWithReferences: ResourceInfoWithReferences[] = + referenceErrors.map(ref => ({...ref.resource, ...{references: ref.references}})); + const data = { + multiple: true, + resources: imagesWithReferences, + configuration: { + title: 'image.images-are-in-use', + message: this.translate.instant('image.images-are-in-use-text'), + deleteText: 'image.delete-image-in-use-text', + selectedText: 'image.selected-images', + columns: ['select', 'preview', 'title', 'references'], + datasource: new ImagesDatasource(null, imagesWithReferences, entity => true) + } + }; + this.dialog.open(ResourcesInUseDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], - data: { - multiple: true, - images: imagesWithReferences - } + data }).afterClosed().subscribe((forceDeleteImages) => { if (forceDeleteImages && forceDeleteImages.length) { const forceDeleteTasks = forceDeleteImages.map((image) => diff --git a/ui-ngx/src/app/shared/components/image/image-references.component.ts b/ui-ngx/src/app/shared/components/image/image-references.component.ts index af2b4e9798..802408ee3a 100644 --- a/ui-ngx/src/app/shared/components/image/image-references.component.ts +++ b/ui-ngx/src/app/shared/components/image/image-references.component.ts @@ -17,7 +17,7 @@ import { ChangeDetectorRef, Component, Input, OnInit } from '@angular/core'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { ImageReferences } from '@shared/models/resource.models'; +import { ResourceReferences } from '@shared/models/resource.models'; import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; import { TranslateService } from '@ngx-translate/core'; import { getEntityDetailsPageURL } from '@core/utils'; @@ -54,7 +54,7 @@ type ReferencedEntitiesEntry = [string, TenantReferencedEntities]; export class ImageReferencesComponent implements OnInit { @Input() - references: ImageReferences; + references: ResourceReferences; popoverComponent: TbPopoverComponent; @@ -99,7 +99,7 @@ export class ImageReferencesComponent implements OnInit { return tenantId === NULL_UUID; } - private hasNonSystemEntities(references: ImageReferences): boolean { + private hasNonSystemEntities(references: ResourceReferences): boolean { for (const entityTypeStr of Object.keys(references)) { const entities = this.references[entityTypeStr]; if (entities.some(e => e.tenantId && e.tenantId.id && e.tenantId.id !== NULL_UUID)) { @@ -109,7 +109,7 @@ export class ImageReferencesComponent implements OnInit { return false; } - private toReferencedEntitiesList(references: ImageReferences): ReferencedEntityInfo[] { + private toReferencedEntitiesList(references: ResourceReferences): ReferencedEntityInfo[] { const result: ReferencedEntityInfo[] = []; for (const entityTypeStr of Object.keys(references)) { const entityType = entityTypeStr as EntityType; @@ -127,7 +127,7 @@ export class ImageReferencesComponent implements OnInit { return result; } - private toReferencedEntitiesEntries(references: ImageReferences): Observable { + private toReferencedEntitiesEntries(references: ResourceReferences): Observable { let referencedEntities: ReferencedEntities = {}; const referencedEntitiesList = this.toReferencedEntitiesList(references); for (const referencedEntityInfo of referencedEntitiesList) { diff --git a/ui-ngx/src/app/shared/components/js-func.component.ts b/ui-ngx/src/app/shared/components/js-func.component.ts index 504823d40a..7c7d14eff5 100644 --- a/ui-ngx/src/app/shared/components/js-func.component.ts +++ b/ui-ngx/src/app/shared/components/js-func.component.ts @@ -20,9 +20,11 @@ import { ElementRef, forwardRef, Input, + OnChanges, OnDestroy, OnInit, Renderer2, + SimpleChanges, ViewChild, ViewContainerRef, ViewEncapsulation @@ -35,7 +37,7 @@ import { ActionNotificationHide, ActionNotificationShow } from '@core/notificati import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { UtilsService } from '@core/services/utils.service'; -import { deepClone, guid, isUndefined, isUndefinedOrNull } from '@app/core/utils'; +import { deepClone, guid, isEqual, isObject, isUndefined, isUndefinedOrNull } from '@app/core/utils'; import { TranslateService } from '@ngx-translate/core'; import { CancelAnimationFrame, RafService } from '@core/services/raf.service'; import { TbEditorCompleter } from '@shared/models/ace/completion.models'; @@ -67,7 +69,7 @@ import { catchError } from 'rxjs/operators'; ], encapsulation: ViewEncapsulation.None }) -export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, Validator { +export class JsFuncComponent implements OnInit, OnChanges, OnDestroy, ControlValueAccessor, Validator { @ViewChild('javascriptEditor', {static: true}) javascriptEditorElmRef: ElementRef; @@ -178,28 +180,11 @@ export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, } ngOnInit(): void { - if (this.functionTitle || this.label) { - this.hideBrackets = true; - } if (!this.resultType || this.resultType.length === 0) { this.resultType = 'nocheck'; } - if (this.functionArgs) { - this.functionArgs.forEach((functionArg) => { - if (this.functionArgsString.length > 0) { - this.functionArgsString += ', '; - } - this.functionArgsString += functionArg; - }); - } - if (this.functionTitle) { - this.functionLabel = `${this.functionTitle}: f(${this.functionArgsString})`; - } else if (this.label) { - this.functionLabel = this.label; - } else { - this.functionLabel = - `function ${this.functionName ? this.functionName : ''}(${this.functionArgsString})${this.hideBrackets ? '' : ' {'}`; - } + this.updateFunctionArgsString() + this.updateFunctionLabel(); const editorElement = this.javascriptEditorElmRef.nativeElement; let editorOptions: Partial = { mode: 'ace/mode/javascript', @@ -251,21 +236,7 @@ export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, } }); } - // @ts-ignore - if (!!this.highlightRules && !!this.jsEditor.session.$mode) { - // @ts-ignore - const newMode = new this.jsEditor.session.$mode.constructor(); - newMode.$highlightRules = new newMode.HighlightRules(); - for(const group in this.highlightRules) { - if(!!newMode.$highlightRules.$rules[group]) { - newMode.$highlightRules.$rules[group].unshift(...this.highlightRules[group]); - } else { - newMode.$highlightRules.$rules[group] = this.highlightRules[group]; - } - } - // @ts-ignore - this.jsEditor.session.$onChangeMode(newMode); - } + this.updateHighlightRules(); this.updateJsWorkerGlobals(); this.initialCompleters = this.jsEditor.completers || []; this.updateCompleters(); @@ -277,6 +248,16 @@ export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, ); } + ngOnChanges(changes: SimpleChanges): void { + for (const propName of Object.keys(changes)) { + const { firstChange, currentValue, previousValue } = changes[propName]; + const isChanged = isObject(currentValue) ? !isEqual(currentValue, previousValue) : currentValue !== previousValue; + if (!firstChange && isChanged) { + this.updateByChangesPropName(propName); + } + } + } + ngOnDestroy(): void { if (this.editorResize$) { this.editorResize$.disconnect(); @@ -329,6 +310,32 @@ export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, ); } + private updateFunctionArgsString(): void { + this.functionArgsString = ''; + if (this.functionArgs) { + this.functionArgsString = this.functionArgs.join(', '); + } + } + + private updateFunctionLabel(): void { + if (this.functionTitle || this.label) { + this.hideBrackets = true; + } + if (this.functionTitle) { + this.functionLabel = `${this.functionTitle}: f(${this.functionArgsString})`; + } else if (this.label) { + this.functionLabel = this.label; + } else { + this.functionLabel = + `function ${this.functionName ? this.functionName : ''}(${this.functionArgsString})${this.hideBrackets ? '' : ' {'}`; + } + this.cd.markForCheck(); + } + + private updatedScriptLanguage() { + this.jsEditor.session.setMode(`ace/mode/${ScriptLanguage.TBEL === this.scriptLanguage ? 'tbel' : 'javascript'}`); + } + validateOnSubmit(): Observable { if (!this.disabled) { this.cleanupJsErrors(); @@ -539,6 +546,52 @@ export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, } } + private updateByChangesPropName(propName: string): void { + switch (propName) { + case 'functionArgs': + this.updateFunctionArgsString() + this.updateFunctionLabel(); + this.updateJsWorkerGlobals(); + break; + case 'label': + case 'functionTitle': + case 'functionName': + this.updateFunctionLabel(); + break; + case 'scriptLanguage': + this.updatedScriptLanguage(); + break; + case 'disableUndefinedCheck': + case 'globalVariables': + this.updateJsWorkerGlobals(); + break; + case 'editorCompleter': + this.updateCompleters(); + break; + case 'highlightRules': + this.updateHighlightRules(); + break; + } + } + + private updateHighlightRules(): void { + // @ts-ignore + if (!!this.highlightRules && !!this.jsEditor.session.$mode) { + // @ts-ignore + const newMode = new this.jsEditor.session.$mode.constructor(); + newMode.$highlightRules = new newMode.HighlightRules(); + for(const group in this.highlightRules) { + if(!!newMode.$highlightRules.$rules[group]) { + newMode.$highlightRules.$rules[group].unshift(...this.highlightRules[group]); + } else { + newMode.$highlightRules.$rules[group] = this.highlightRules[group]; + } + } + // @ts-ignore + this.jsEditor.session.$onChangeMode(newMode); + } + } + private updateJsWorkerGlobals() { // @ts-ignore if (!!this.jsEditor.session.$worker) { diff --git a/ui-ngx/src/app/shared/components/image/images-in-use-dialog.component.html b/ui-ngx/src/app/shared/components/resource/resources-in-use-dialog.component.html similarity index 71% rename from ui-ngx/src/app/shared/components/image/images-in-use-dialog.component.html rename to ui-ngx/src/app/shared/components/resource/resources-in-use-dialog.component.html index 4f4667ceb8..08a35aa412 100644 --- a/ui-ngx/src/app/shared/components/image/images-in-use-dialog.component.html +++ b/ui-ngx/src/app/shared/components/resource/resources-in-use-dialog.component.html @@ -15,10 +15,10 @@ limitations under the License. --> -

    {{title}}

    -
    -
    - +

    {{configuration.title}}

    +
    +
    +
    @@ -28,14 +28,14 @@ [indeterminate]="dataSource.selection.hasValue() && (dataSource.isAllSelected() | async) === false"> - + + (change)="$event ? dataSource.selection.toggle(resource) : null" + [checked]="dataSource.selection.isSelected(resource)"> - + {{ image.title }} @@ -43,30 +43,30 @@ - - {{ 'image.name' | translate }} + + {{ 'resource.title' | translate }} - - {{ translate.get('image.selected-images', {count: dataSource.selection.selected.length}) | async }} + + {{ translate.get(configuration.selectedText, {count: dataSource.selection.selected.length}) | async }} - - {{ image.title }} + + {{ resource.title }} - + + (click)="toggleShowReferences($event, resource, showReferencesButton)">{{ 'image.references' | translate }} - + + [class.mat-selected]="dataSource.selection.isSelected(resource)" + *matRowDef="let resource; columns: configuration.columns;">
    @@ -76,7 +76,7 @@
    - + -
    +
    diff --git a/ui-ngx/src/app/shared/components/image/images-in-use-dialog.component.scss b/ui-ngx/src/app/shared/components/resource/resources-in-use-dialog.component.scss similarity index 96% rename from ui-ngx/src/app/shared/components/image/images-in-use-dialog.component.scss rename to ui-ngx/src/app/shared/components/resource/resources-in-use-dialog.component.scss index 5c053e984f..380a6070eb 100644 --- a/ui-ngx/src/app/shared/components/image/images-in-use-dialog.component.scss +++ b/ui-ngx/src/app/shared/components/resource/resources-in-use-dialog.component.scss @@ -14,7 +14,7 @@ * limitations under the License. */ :host { - .tb-images-in-use-content { + .tb-resources-in-use-content { display: flex; flex-direction: column; gap: 24px; diff --git a/ui-ngx/src/app/shared/components/image/images-in-use-dialog.component.ts b/ui-ngx/src/app/shared/components/resource/resources-in-use-dialog.component.ts similarity index 66% rename from ui-ngx/src/app/shared/components/image/images-in-use-dialog.component.ts rename to ui-ngx/src/app/shared/components/resource/resources-in-use-dialog.component.ts index 681e9a2c53..6f83fded7d 100644 --- a/ui-ngx/src/app/shared/components/image/images-in-use-dialog.component.ts +++ b/ui-ngx/src/app/shared/components/resource/resources-in-use-dialog.component.ts @@ -20,37 +20,50 @@ import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { DialogComponent } from '@shared/components/dialog.component'; import { Router } from '@angular/router'; -import { ImageReferences, ImageResourceInfo, ImageResourceInfoWithReferences } from '@shared/models/resource.models'; -import { ImagesDatasource } from '@shared/components/image/images-datasource'; +import { + ResourceReferences, + ResourceInfoWithReferences, + ResourceInfo +} from '@shared/models/resource.models'; import { MatButton } from '@angular/material/button'; import { TbPopoverService } from '@shared/components/popover.service'; import { ImageReferencesComponent } from '@shared/components/image/image-references.component'; import { TranslateService } from '@ngx-translate/core'; +import { Datasource } from "@shared/models/widget.models"; -export interface ImagesInUseDialogData { +interface ResourcesInUseDialogDataConfiguration { + title: string; + message: string; + columns: string[]; + deleteText: string; + selectedText: string; + datasource?: Datasource; +} + +export interface ResourcesInUseDialogData { multiple: boolean; - images: ImageResourceInfoWithReferences[]; + resources: ResourceInfoWithReferences[]; + configuration: ResourcesInUseDialogDataConfiguration; } @Component({ - selector: 'tb-images-in-use-dialog', - templateUrl: './images-in-use-dialog.component.html', - styleUrls: ['./images-in-use-dialog.component.scss'] + selector: 'tb-resources-in-use-dialog', + templateUrl: './resources-in-use-dialog.component.html', + styleUrls: ['./resources-in-use-dialog.component.scss'] }) -export class ImagesInUseDialogComponent extends - DialogComponent implements OnInit { - - title: string; - message: string; +export class ResourcesInUseDialogComponent extends + DialogComponent implements OnInit { - references: ImageReferences; + displayPreview: boolean; + configuration: ResourcesInUseDialogDataConfiguration; + references: ResourceReferences; - dataSource: ImagesDatasource; + dataSource: Datasource; constructor(protected store: Store, protected router: Router, - @Inject(MAT_DIALOG_DATA) public data: ImagesInUseDialogData, - public dialogRef: MatDialogRef, + @Inject(MAT_DIALOG_DATA) public data: ResourcesInUseDialogData, + public dialogRef: MatDialogRef, public translate: TranslateService, private renderer: Renderer2, private viewContainerRef: ViewContainerRef, @@ -59,14 +72,12 @@ export class ImagesInUseDialogComponent extends } ngOnInit(): void { + this.configuration = this.data.configuration; + this.displayPreview = this.data.configuration.columns.includes('preview'); if (this.data.multiple) { - this.title = this.translate.instant('image.images-are-in-use'); - this.message = this.translate.instant('image.images-are-in-use-text'); - this.dataSource = new ImagesDatasource(null, this.data.images, entity => true); + this.dataSource = this.data.configuration.datasource; } else { - this.title = this.translate.instant('image.image-is-in-use'); - this.message = this.translate.instant('image.image-is-in-use-text', {title: this.data.images[0].title}); - this.references = this.data.images[0].references; + this.references = this.data.resources[0].references; } } @@ -78,11 +89,11 @@ export class ImagesInUseDialogComponent extends if (this.data.multiple) { this.dialogRef.close(this.dataSource.selection.selected); } else { - this.dialogRef.close(this.data.images); + this.dialogRef.close(this.data.resources); } } - toggleShowReferences($event: Event, image: ImageResourceInfoWithReferences, referencesButton: MatButton) { + toggleShowReferences($event: Event, resource: ResourceInfoWithReferences, referencesButton: MatButton) { if ($event) { $event.stopPropagation(); } @@ -93,7 +104,7 @@ export class ImagesInUseDialogComponent extends const referencesPopover = this.popoverService.displayPopover(trigger, this.renderer, this.viewContainerRef, ImageReferencesComponent, 'top', true, null, { - references: image.references + references: resource.references }, {}, {}, {}, false, visible => { diff --git a/ui-ngx/src/app/shared/components/value-input.component.html b/ui-ngx/src/app/shared/components/value-input.component.html index e8409d869e..1664a4af95 100644 --- a/ui-ngx/src/app/shared/components/value-input.component.html +++ b/ui-ngx/src/app/shared/components/value-input.component.html @@ -32,8 +32,8 @@ - + - + - +
    - + -
    diff --git a/ui-ngx/src/app/shared/components/value-input.component.ts b/ui-ngx/src/app/shared/components/value-input.component.ts index 312e12a2a7..07bdaaa742 100644 --- a/ui-ngx/src/app/shared/components/value-input.component.ts +++ b/ui-ngx/src/app/shared/components/value-input.component.ts @@ -81,6 +81,14 @@ export class ValueInputComponent implements OnInit, OnDestroy, OnChanges, Contro @coerceBoolean() shortBooleanField = false; + @Input() + @coerceBoolean() + required = true; + + @Input() + @coerceBoolean() + hideJsonEdit = false; + @Input() layout: ValueInputLayout | Layout = 'row'; diff --git a/ui-ngx/src/app/shared/import-export/import-export.service.ts b/ui-ngx/src/app/shared/import-export/import-export.service.ts index 5212c15fd4..022a844ceb 100644 --- a/ui-ngx/src/app/shared/import-export/import-export.service.ts +++ b/ui-ngx/src/app/shared/import-export/import-export.service.ts @@ -88,6 +88,8 @@ import { ExportResourceDialogDialogResult } from '@shared/import-export/export-resource-dialog.component'; import { FormProperty, propertyValid } from '@shared/models/dynamic-form.models'; +import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; +import { CalculatedField } from '@shared/models/calculated-field.models'; export type editMissingAliasesFunction = (widgets: Array, isSingleWidget: boolean, customTitle: string, missingEntityAliases: EntityAliases) => Observable; @@ -116,6 +118,7 @@ export class ImportExportService { private imageService: ImageService, private utils: UtilsService, private itembuffer: ItemBufferService, + private calculatedFieldsService: CalculatedFieldsService, private dialog: MatDialog) { } @@ -171,6 +174,35 @@ export class ImportExportService { ); } + public exportCalculatedField(calculatedFieldId: string): void { + this.calculatedFieldsService.getCalculatedFieldById(calculatedFieldId).subscribe({ + next: (calculatedField) => { + let name = calculatedField.name; + name = name.toLowerCase().replace(/\W/g, '_'); + this.exportToPc(this.prepareCalculatedFieldExport(calculatedField), name); + }, + error: (e) => { + this.handleExportError(e, 'calculated-fields.export-failed-error'); + } + }); + } + + public importCalculatedField(entityId: EntityId): Observable { + return this.openImportDialog('calculated-fields.import', 'calculated-fields.file').pipe( + mergeMap((calculatedField: CalculatedField) => { + if (!this.validateImportedCalculatedField({ entityId, ...calculatedField })) { + this.store.dispatch(new ActionNotificationShow( + {message: this.translate.instant('calculated-fields.invalid-file-error'), + type: 'error'})); + throw new Error('Invalid calculated field file'); + } else { + return this.calculatedFieldsService.saveCalculatedField(this.prepareImport({ entityId, ...calculatedField })); + } + }), + catchError(() => of(null)), + ); + } + public exportDashboard(dashboardId: string) { this.getIncludeResourcesPreference('includeResourcesInExportDashboard').subscribe(includeResources => { this.openExportDialog('dashboard.export', 'dashboard.export-prompt', includeResources).subscribe(result => { @@ -957,6 +989,16 @@ export class ImportExportService { } } + private validateImportedCalculatedField(calculatedField: CalculatedField): boolean { + const { name, configuration, entityId } = calculatedField; + return isNotEmptyStr(name) + && isDefined(configuration) + && isDefined(entityId?.id) + && !!Object.keys(configuration.arguments).length + && isDefined(configuration.expression) + && isDefined(configuration.output) + } + private validateImportedImage(image: ImageExportData): boolean { return !(!isNotEmptyStr(image.data) || !isNotEmptyStr(image.title) @@ -1209,6 +1251,11 @@ export class ImportExportService { return profile; } + private prepareCalculatedFieldExport(calculatedField: CalculatedField): CalculatedField { + delete calculatedField.entityId; + return this.prepareExport(calculatedField); + } + private prepareExport(data: any): any { const exportedData = deepClone(data); if (isDefined(exportedData.id)) { diff --git a/ui-ngx/src/app/shared/models/ace/ace.models.ts b/ui-ngx/src/app/shared/models/ace/ace.models.ts index f6af2d6a52..5287886b84 100644 --- a/ui-ngx/src/app/shared/models/ace/ace.models.ts +++ b/ui-ngx/src/app/shared/models/ace/ace.models.ts @@ -365,5 +365,16 @@ export interface AceHighlightRule { next?: string; } +export const dotOperatorHighlightRule: AceHighlightRule = { + token: 'punctuation.operator', + regex: /[.](?![.])/, +}; + +export const endGroupHighlightRule: AceHighlightRule = { + regex: '', + token: 'empty', + next: 'no_regex' +}; + diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts new file mode 100644 index 0000000000..f6d7aaaa1e --- /dev/null +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -0,0 +1,551 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + AdditionalDebugActionConfig, + HasEntityDebugSettings, + HasTenantId, + HasVersion +} from '@shared/models/entity.models'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; +import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; +import { EntityId } from '@shared/models/id/entity-id'; +import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; +import { EntityType } from '@shared/models/entity-type.models'; +import { AliasFilterType } from '@shared/models/alias.models'; +import { Observable } from 'rxjs'; +import { TbEditorCompleter } from '@shared/models/ace/completion.models'; +import { + AceHighlightRule, + AceHighlightRules, + dotOperatorHighlightRule, + endGroupHighlightRule +} from '@shared/models/ace/ace.models'; + +export interface CalculatedField extends Omit, 'label'>, HasVersion, HasEntityDebugSettings, HasTenantId, ExportableEntity { + configuration: CalculatedFieldConfiguration; + type: CalculatedFieldType; + entityId: EntityId; +} + +export enum CalculatedFieldType { + SIMPLE = 'SIMPLE', + SCRIPT = 'SCRIPT', +} + +export const CalculatedFieldTypeTranslations = new Map( + [ + [CalculatedFieldType.SIMPLE, 'calculated-fields.type.simple'], + [CalculatedFieldType.SCRIPT, 'calculated-fields.type.script'], + ] +) + +export interface CalculatedFieldConfiguration { + type: CalculatedFieldType; + expression: string; + arguments: Record; + output: CalculatedFieldOutput; +} + +export interface CalculatedFieldOutput { + type: OutputType; + name: string; + scope?: AttributeScope; +} + +export enum ArgumentEntityType { + Current = 'CURRENT', + Device = 'DEVICE', + Asset = 'ASSET', + Customer = 'CUSTOMER', + Tenant = 'TENANT', +} + +export const ArgumentEntityTypeTranslations = new Map( + [ + [ArgumentEntityType.Current, 'calculated-fields.argument-current'], + [ArgumentEntityType.Device, 'calculated-fields.argument-device'], + [ArgumentEntityType.Asset, 'calculated-fields.argument-asset'], + [ArgumentEntityType.Customer, 'calculated-fields.argument-customer'], + [ArgumentEntityType.Tenant, 'calculated-fields.argument-tenant'], + ] +) + +export enum ArgumentType { + Attribute = 'ATTRIBUTE', + LatestTelemetry = 'TS_LATEST', + Rolling = 'TS_ROLLING', +} + +export enum TestArgumentType { + Single = 'SINGLE_VALUE', + Rolling = 'TS_ROLLING', +} + +export const TestArgumentTypeMap = new Map( + [ + [ArgumentType.Attribute, TestArgumentType.Single], + [ArgumentType.LatestTelemetry, TestArgumentType.Single], + [ArgumentType.Rolling, TestArgumentType.Rolling], + ] +) + +export enum OutputType { + Attribute = 'ATTRIBUTES', + Timeseries = 'TIME_SERIES', +} + +export const OutputTypeTranslations = new Map( + [ + [OutputType.Attribute, 'calculated-fields.attribute'], + [OutputType.Timeseries, 'calculated-fields.timeseries'], + ] +) + +export const ArgumentTypeTranslations = new Map( + [ + [ArgumentType.Attribute, 'calculated-fields.attribute'], + [ArgumentType.LatestTelemetry, 'calculated-fields.latest-telemetry'], + [ArgumentType.Rolling, 'calculated-fields.rolling'], + ] +) + +export interface CalculatedFieldArgument { + refEntityKey: RefEntityKey; + defaultValue?: string; + refEntityId?: RefEntityKey; + limit?: number; + timeWindow?: number; +} + +export interface RefEntityKey { + key: string; + type: ArgumentType; + scope?: AttributeScope; +} + +export interface RefEntityKey { + entityType: ArgumentEntityType; + id: string; +} + +export interface CalculatedFieldArgumentValue extends CalculatedFieldArgument { + argumentName: string; +} + +export type CalculatedFieldTestScriptFn = (calculatedField: CalculatedField, argumentsObj?: Record, closeAllOnSave?: boolean) => Observable; + +export interface CalculatedFieldDialogData { + value?: CalculatedField; + buttonTitle: string; + entityId: EntityId; + debugLimitsConfiguration: string; + tenantId: string; + entityName?: string; + additionalDebugActionConfig: AdditionalDebugActionConfig<(calculatedField: CalculatedField) => void>; + getTestScriptDialogFn: CalculatedFieldTestScriptFn; + isDirty?: boolean; +} + +export interface CalculatedFieldDebugDialogData { + tenantId: string; + value: CalculatedField; + getTestScriptDialogFn: CalculatedFieldTestScriptFn; +} + +export interface CalculatedFieldTestScriptInputParams { + arguments: CalculatedFieldEventArguments; + expression: string; +} + +export interface CalculatedFieldTestScriptDialogData extends CalculatedFieldTestScriptInputParams { + argumentsEditorCompleter: TbEditorCompleter; + argumentsHighlightRules: AceHighlightRules; + openCalculatedFieldEdit?: boolean; +} + +export interface ArgumentEntityTypeParams { + title: string; + entityType: EntityType +} + +export const ArgumentEntityTypeParamsMap =new Map([ + [ArgumentEntityType.Device, { title: 'calculated-fields.device-name', entityType: EntityType.DEVICE }], + [ArgumentEntityType.Asset, { title: 'calculated-fields.asset-name', entityType: EntityType.ASSET }], + [ArgumentEntityType.Customer, { title: 'calculated-fields.customer-name', entityType: EntityType.CUSTOMER }], +]) + +export const getCalculatedFieldCurrentEntityFilter = (entityName: string, entityId: EntityId) => { + switch (entityId.entityType) { + case EntityType.ASSET_PROFILE: + return { + assetTypes: [entityName], + type: AliasFilterType.assetType + }; + case EntityType.DEVICE_PROFILE: + return { + deviceTypes: [entityName], + type: AliasFilterType.deviceType + }; + default: + return { + type: AliasFilterType.singleEntity, + singleEntity: entityId, + }; + } +} + +export interface CalculatedFieldArgumentValueBase { + argumentName: string; + type: ArgumentType; +} + +export interface CalculatedFieldAttributeArgumentValue extends CalculatedFieldArgumentValueBase { + ts: number; + value: ValueType; +} + +export interface CalculatedFieldLatestTelemetryArgumentValue extends CalculatedFieldArgumentValueBase { + ts: number; + value: ValueType; +} + +export interface CalculatedFieldRollingTelemetryArgumentValue extends CalculatedFieldArgumentValueBase { + timeWindow: { startTs: number; endTs: number; limit: number }; + values: CalculatedFieldSingleArgumentValue[]; +} + +export type CalculatedFieldSingleArgumentValue = CalculatedFieldAttributeArgumentValue & CalculatedFieldLatestTelemetryArgumentValue; + +export type CalculatedFieldArgumentEventValue = CalculatedFieldAttributeArgumentValue | CalculatedFieldLatestTelemetryArgumentValue | CalculatedFieldRollingTelemetryArgumentValue; + +export type CalculatedFieldEventArguments = Record>; + +export const CalculatedFieldLatestTelemetryArgumentAutocomplete = { + meta: 'object', + type: '{ ts: number; value: any; }', + description: 'Calculated field latest telemetry value argument.', + children: { + ts: { + meta: 'number', + type: 'number', + description: 'Time stamp', + }, + value: { + meta: 'any', + type: 'any', + description: 'Value', + } + }, +}; + +export const CalculatedFieldAttributeValueArgumentAutocomplete = { + meta: 'object', + type: '{ ts: number; value: any; }', + description: 'Calculated field attribute value argument.', + children: { + ts: { + meta: 'number', + type: 'number', + description: 'Time stamp', + }, + value: { + meta: 'any', + type: 'any', + description: 'Value', + } + }, +}; +export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { + max: { + meta: 'function', + description: 'Computes the maximum value in the list of rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + args: [ + { + name: 'ignoreNaN', + description: 'Whether to ignore NaN values. Equals true by default.', + type: 'boolean', + optional: true, + } + ], + return: { + description: 'The maximum value, or NaN if applicable', + type: 'number' + } + }, + min: { + meta: 'function', + description: 'Computes the minimum value in the list of rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + args: [ + { + name: 'ignoreNaN', + description: 'Whether to ignore NaN values. Equals true by default.', + type: 'boolean', + optional: true, + } + ], + return: { + description: 'The minimum value, or NaN if applicable', + type: 'number' + } + }, + mean: { + meta: 'function', + description: 'Computes the mean value of the rolling argument values list. Returns NaN if any value is NaN and ignoreNaN is false.', + args: [ + { + name: 'ignoreNaN', + description: 'Whether to ignore NaN values. Equals true by default.', + type: 'boolean', + optional: true, + } + ], + return: { + description: 'The mean value, or NaN if applicable', + type: 'number' + } + }, + std: { + meta: 'function', + description: 'Computes the standard deviation in the list of rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + args: [ + { + name: 'ignoreNaN', + description: 'Whether to ignore NaN values. Equals true by default.', + type: 'boolean', + optional: true, + } + ], + return: { + description: 'The standard deviation, or NaN if applicable', + type: 'number' + } + }, + median: { + meta: 'function', + description: 'Computes the median value of the rolling argument values list. Returns NaN if any value is NaN and ignoreNaN is false.', + args: [ + { + name: 'ignoreNaN', + description: 'Whether to ignore NaN values. Equals true by default.', + type: 'boolean', + optional: true, + } + ], + return: { + description: 'The median value, or NaN if applicable', + type: 'number' + } + }, + count: { + meta: 'function', + description: 'Counts values in the list of rolling argument values. Counts non-NaN values if ignoreNaN is true, otherwise - total size.', + args: [ + { + name: 'ignoreNaN', + description: 'Whether to ignore NaN values. Equals true by default.', + type: 'boolean', + optional: true, + } + ], + return: { + description: 'The count of values', + type: 'number' + } + }, + last: { + meta: 'function', + description: 'Returns the last non-NaN value in the list of rolling argument values if ignoreNaN is true, otherwise - the last value.', + args: [ + { + name: 'ignoreNaN', + description: 'Whether to ignore NaN values. Equals true by default.', + type: 'boolean', + optional: true, + } + ], + return: { + description: 'The last value, or NaN if applicable', + type: 'number' + } + }, + first: { + meta: 'function', + description: 'Returns the first non-NaN value in the list of rolling argument values if ignoreNaN is true, otherwise - the first value.', + args: [ + { + name: 'ignoreNaN', + description: 'Whether to ignore NaN values. Equals true by default.', + type: 'boolean', + optional: true, + } + ], + return: { + description: 'The first value, or NaN if applicable', + type: 'number' + } + }, + sum: { + meta: 'function', + description: 'Computes the sum of values in the list of rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + args: [ + { + name: 'ignoreNaN', + description: 'Whether to ignore NaN values. Equals true by default.', + type: 'boolean', + optional: true, + } + ], + return: { + description: 'The sum of values, or NaN if applicable', + type: 'number' + } + } +}; + +export const CalculatedFieldRollingValueArgumentAutocomplete = { + meta: 'object', + type: '{ values: { ts: number; value: any; }[]; timeWindow: { startTs: number; endTs: number; limit: number } }; }', + description: 'Calculated field rolling value argument.', + children: { + ...CalculatedFieldRollingValueArgumentFunctionsAutocomplete, + values: { + meta: 'array', + type: '{ ts: number; value: any; }[]', + description: 'Values array', + }, + timeWindow: { + meta: 'object', + type: '{ startTs: number; endTs: number; limit: number }', + description: 'Time window configuration', + children: { + startTs: { + meta: 'number', + type: 'number', + description: 'Start time stamp', + }, + endTs: { + meta: 'number', + type: 'number', + description: 'End time stamp', + }, + limit: { + meta: 'number', + type: 'number', + description: 'Limit', + } + } + } + }, +}; + +export const getCalculatedFieldArgumentsEditorCompleter = (argumentsObj: Record): TbEditorCompleter => { + return new TbEditorCompleter(Object.keys(argumentsObj).reduce((acc, key) => { + switch (argumentsObj[key].refEntityKey.type) { + case ArgumentType.Attribute: + acc[key] = CalculatedFieldAttributeValueArgumentAutocomplete; + break; + case ArgumentType.LatestTelemetry: + acc[key] = CalculatedFieldLatestTelemetryArgumentAutocomplete; + break; + case ArgumentType.Rolling: + acc[key] = CalculatedFieldRollingValueArgumentAutocomplete; + break; + } + return acc; + }, {})); +} + +export const getCalculatedFieldArgumentsHighlights = ( + argumentsObj: Record +): AceHighlightRules => { + return { + start: Object.keys(argumentsObj).map(key => ({ + token: 'tb.calculated-field-key', + regex: `\\b${key}\\b`, + next: argumentsObj[key].refEntityKey.type === ArgumentType.Rolling + ? 'calculatedFieldRollingArgumentValue' + : 'calculatedFieldSingleArgumentValue' + })), + ...calculatedFieldSingleArgumentValueHighlightRules, + ...calculatedFieldRollingArgumentValueHighlightRules, + ...calculatedFieldTimeWindowArgumentValueHighlightRules + }; +}; + +const calculatedFieldSingleArgumentValueHighlightRules: AceHighlightRules = { + calculatedFieldSingleArgumentValue: [ + dotOperatorHighlightRule, + { + token: 'tb.calculated-field-value', + regex: /value/, + next: 'no_regex' + }, + { + token: 'tb.calculated-field-ts', + regex: /ts/, + next: 'no_regex' + }, + endGroupHighlightRule + ], +} + +const calculatedFieldRollingArgumentValueFunctionsHighlightRules: Array = + ['max', 'min', 'mean', 'std', 'median', 'count', 'last', 'first', 'sum'].map(funcName => ({ + token: 'tb.calculated-field-func', + regex: `\\b${funcName}\\b`, + next: 'no_regex' + })); + +const calculatedFieldRollingArgumentValueHighlightRules: AceHighlightRules = { + calculatedFieldRollingArgumentValue: [ + dotOperatorHighlightRule, + { + token: 'tb.calculated-field-values', + regex: /values/, + next: 'no_regex' + }, + { + token: 'tb.calculated-field-time-window', + regex: /timeWindow/, + next: 'calculatedFieldRollingArgumentTimeWindow' + }, + ...calculatedFieldRollingArgumentValueFunctionsHighlightRules, + endGroupHighlightRule + ], +} + +const calculatedFieldTimeWindowArgumentValueHighlightRules: AceHighlightRules = { + calculatedFieldRollingArgumentTimeWindow: [ + dotOperatorHighlightRule, + { + token: 'tb.calculated-field-start-ts', + regex: /startTs/, + next: 'no_regex' + }, + { + token: 'tb.calculated-field-end-ts', + regex: /endTs/, + next: 'no_regex' + }, + { + token: 'tb.calculated-field-limit', + regex: /limit/, + next: 'no_regex' + }, + endGroupHighlightRule + ] +} diff --git a/ui-ngx/src/app/shared/models/constants.ts b/ui-ngx/src/app/shared/models/constants.ts index c811d11052..bdd61e060b 100644 --- a/ui-ngx/src/app/shared/models/constants.ts +++ b/ui-ngx/src/app/shared/models/constants.ts @@ -196,6 +196,7 @@ export const HelpLinks = { mobileApplication: `${helpBaseUrl}/docs${docPlatformPrefix}/mobile-center/applications/`, mobileBundle: `${helpBaseUrl}/docs${docPlatformPrefix}/mobile-center/mobile-center/`, mobileQrCode: `${helpBaseUrl}/docs${docPlatformPrefix}/user-guide/ui/mobile-qr-code/`, + calculatedField: `${helpBaseUrl}/docs${docPlatformPrefix}/`, } }; /* eslint-enable max-len */ diff --git a/ui-ngx/src/app/shared/models/entity-type.models.ts b/ui-ngx/src/app/shared/models/entity-type.models.ts index fe344352c3..48cad42e7b 100644 --- a/ui-ngx/src/app/shared/models/entity-type.models.ts +++ b/ui-ngx/src/app/shared/models/entity-type.models.ts @@ -49,7 +49,8 @@ export enum EntityType { OAUTH2_CLIENT = 'OAUTH2_CLIENT', DOMAIN = 'DOMAIN', MOBILE_APP_BUNDLE = 'MOBILE_APP_BUNDLE', - MOBILE_APP = 'MOBILE_APP' + MOBILE_APP = 'MOBILE_APP', + CALCULATED_FIELD = 'CALCULATED_FIELD', } export enum AliasEntityType { @@ -478,6 +479,18 @@ export const entityTypeTranslations = new Map void> { + action: Action; + title: string; +} + export type VersionedEntity = EntityInfoData & HasVersion | RuleChainMetaData; diff --git a/ui-ngx/src/app/shared/models/event.models.ts b/ui-ngx/src/app/shared/models/event.models.ts index 30f9a57b89..027fae1aa9 100644 --- a/ui-ngx/src/app/shared/models/event.models.ts +++ b/ui-ngx/src/app/shared/models/event.models.ts @@ -29,7 +29,8 @@ export enum EventType { export enum DebugEventType { DEBUG_RULE_NODE = 'DEBUG_RULE_NODE', - DEBUG_RULE_CHAIN = 'DEBUG_RULE_CHAIN' + DEBUG_RULE_CHAIN = 'DEBUG_RULE_CHAIN', + DEBUG_CALCULATED_FIELD = 'DEBUG_CALCULATED_FIELD' } export const eventTypeTranslations = new Map( @@ -39,6 +40,7 @@ export const eventTypeTranslations = new Map [EventType.STATS, 'event.type-stats'], [DebugEventType.DEBUG_RULE_NODE, 'event.type-debug-rule-node'], [DebugEventType.DEBUG_RULE_CHAIN, 'event.type-debug-rule-chain'], + [DebugEventType.DEBUG_CALCULATED_FIELD, 'event.type-debug-calculated-field'], ] ); @@ -80,7 +82,7 @@ export interface DebugRuleChainEventBody extends BaseEventBody { error?: string; } -export type EventBody = ErrorEventBody & LcEventEventBody & StatsEventBody & DebugRuleNodeEventBody & DebugRuleChainEventBody; +export type EventBody = ErrorEventBody & LcEventEventBody & StatsEventBody & DebugRuleNodeEventBody & DebugRuleChainEventBody & CalculatedFieldEventBody; export interface Event extends BaseData { tenantId: TenantId; @@ -90,6 +92,16 @@ export interface Event extends BaseData { body: EventBody; } +export interface CalculatedFieldEventBody extends BaseFilterEventBody { + calculatedFieldId: string; + entityId: string; + entityType: EntityType; + arguments: string, + result: string, + msgId: string; + msgType: string; +} + export interface BaseFilterEventBody { server?: string; } diff --git a/ui-ngx/src/app/shared/models/id/calculated-field-id.ts b/ui-ngx/src/app/shared/models/id/calculated-field-id.ts new file mode 100644 index 0000000000..f42e20f3d2 --- /dev/null +++ b/ui-ngx/src/app/shared/models/id/calculated-field-id.ts @@ -0,0 +1,26 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { EntityId } from './entity-id'; +import { EntityType } from '@shared/models/entity-type.models'; + +export class CalculatedFieldId implements EntityId { + entityType = EntityType.CALCULATED_FIELD; + id: string; + constructor(id: string) { + this.id = id; + } +} diff --git a/ui-ngx/src/app/shared/models/public-api.ts b/ui-ngx/src/app/shared/models/public-api.ts index 470b628800..53e4bb286e 100644 --- a/ui-ngx/src/app/shared/models/public-api.ts +++ b/ui-ngx/src/app/shared/models/public-api.ts @@ -61,3 +61,4 @@ export * from './widgets-bundle.model'; export * from './window-message.model'; export * from './usage.models'; export * from './query/query.models'; +export * from './regex.constants'; diff --git a/ui-ngx/src/app/shared/models/regex.constants.ts b/ui-ngx/src/app/shared/models/regex.constants.ts new file mode 100644 index 0000000000..f0489060ef --- /dev/null +++ b/ui-ngx/src/app/shared/models/regex.constants.ts @@ -0,0 +1,19 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +export const noLeadTrailSpacesRegex = /^\S+(?: \S+)*$/; + +export const charsWithNumRegex = /^[a-zA-Z]+[a-zA-Z0-9]*$/; diff --git a/ui-ngx/src/app/shared/models/resource.models.ts b/ui-ngx/src/app/shared/models/resource.models.ts index eda0182383..0419e40a7c 100644 --- a/ui-ngx/src/app/shared/models/resource.models.ts +++ b/ui-ngx/src/app/shared/models/resource.models.ts @@ -120,28 +120,28 @@ export interface ImageExportData { export type ImageResourceType = 'tenant' | 'system'; export type TBResourceScope = 'tenant' | 'system'; -export type ImageReferences = {[entityType: string]: Array & HasTenantId>}; +export type ResourceReferences = {[entityType: string]: Array & HasTenantId>}; -export interface ImageResourceInfoWithReferences extends ImageResourceInfo { - references: ImageReferences; +export interface ResourceInfoWithReferences extends ResourceInfo { + references: ResourceReferences; } -export interface ImageDeleteResult { - image: ImageResourceInfo; +export interface ResourceDeleteResult { + resource: TbResourceInfo; success: boolean; - imageIsReferencedError?: boolean; + resourceIsReferencedError?: boolean; error?: any; - references?: ImageReferences; + references?: ResourceReferences; } -export const toImageDeleteResult = (image: ImageResourceInfo, e?: any): ImageDeleteResult => { +export const toResourceDeleteResult = (resource: ResourceInfo, e?: any): ResourceDeleteResult => { if (!e) { - return {image, success: true}; + return {resource, success: true}; } else { - const result: ImageDeleteResult = {image, success: false, error: e}; + const result: ResourceDeleteResult = {resource, success: false, error: e}; if (e?.status === 400 && e?.error?.success === false && e?.error?.references) { - const references: ImageReferences = e?.error?.references; - result.imageIsReferencedError = true; + const references: ResourceReferences = e?.error?.references; + result.resourceIsReferencedError = true; result.references = references; } return result; diff --git a/ui-ngx/src/app/shared/models/rule-node.models.ts b/ui-ngx/src/app/shared/models/rule-node.models.ts index ad7f3c01d5..32422a87e0 100644 --- a/ui-ngx/src/app/shared/models/rule-node.models.ts +++ b/ui-ngx/src/app/shared/models/rule-node.models.ts @@ -25,7 +25,7 @@ import { AfterViewInit, DestroyRef, Directive, EventEmitter, inject, OnInit } fr import { AbstractControl, UntypedFormGroup } from '@angular/forms'; import { RuleChainType } from '@shared/models/rule-chain.models'; import { DebugRuleNodeEventBody } from '@shared/models/event.models'; -import { HasEntityDebugSettings } from '@shared/models/entity.models'; +import { EntityTestScriptResult, HasEntityDebugSettings } from '@shared/models/entity.models'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; export interface RuleNodeConfiguration { @@ -372,10 +372,7 @@ export interface TestScriptInputParams { msgType: string; } -export interface TestScriptResult { - output: string; - error: string; -} +export type TestScriptResult = EntityTestScriptResult; export enum MessageType { POST_ATTRIBUTES_REQUEST = 'POST_ATTRIBUTES_REQUEST', diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 23c2d95762..38ff63e3c6 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -95,6 +95,13 @@ export interface DefaultTenantProfileConfiguration { rpcTtlDays: number; queueStatsTtlDays: number; ruleEngineExceptionsTtlDays: number; + + maxCalculatedFieldsPerEntity: number; + maxArgumentsPerCF: number; + maxDataPointsPerRollingArg: number; + maxStateSizeInKBytes: number; + maxSingleValueArgumentSizeInKBytes: number; + calculatedFieldDebugEventsRateLimit: string; } export type TenantProfileConfigurations = DefaultTenantProfileConfiguration; @@ -148,7 +155,13 @@ export function createTenantProfileConfiguration(type: TenantProfileType): Tenan alarmsTtlDays: 0, rpcTtlDays: 0, queueStatsTtlDays: 0, - ruleEngineExceptionsTtlDays: 0 + ruleEngineExceptionsTtlDays: 0, + maxCalculatedFieldsPerEntity: 0, + maxArgumentsPerCF: 0, + maxDataPointsPerRollingArg: 0, + maxStateSizeInKBytes: 0, + maxSingleValueArgumentSizeInKBytes: 0, + calculatedFieldDebugEventsRateLimit: '' }; configuration = {...defaultConfiguration, type: TenantProfileType.DEFAULT}; break; diff --git a/ui-ngx/src/app/shared/shared.module.ts b/ui-ngx/src/app/shared/shared.module.ts index 9245673d89..7eb622029f 100644 --- a/ui-ngx/src/app/shared/shared.module.ts +++ b/ui-ngx/src/app/shared/shared.module.ts @@ -201,7 +201,7 @@ import { ImageGalleryComponent } from '@shared/components/image/image-gallery.co import { UploadImageDialogComponent } from '@shared/components/image/upload-image-dialog.component'; import { ImageDialogComponent } from '@shared/components/image/image-dialog.component'; import { ImageReferencesComponent } from '@shared/components/image/image-references.component'; -import { ImagesInUseDialogComponent } from '@shared/components/image/images-in-use-dialog.component'; +import { ResourcesInUseDialogComponent } from '@shared/components/resource/resources-in-use-dialog.component'; import { GalleryImageInputComponent } from '@shared/components/image/gallery-image-input.component'; import { MultipleGalleryImageInputComponent } from '@shared/components/image/multiple-gallery-image-input.component'; import { EmbedImageDialogComponent } from '@shared/components/image/embed-image-dialog.component'; @@ -224,6 +224,7 @@ import { IntervalOptionsConfigPanelComponent } from '@shared/components/time/int import { GroupingIntervalOptionsComponent } from '@shared/components/time/aggregation/grouping-interval-options.component'; import { JsFuncModulesComponent } from '@shared/components/js-func-modules.component'; import { JsFuncModuleRowComponent } from '@shared/components/js-func-module-row.component'; +import { EntityKeyAutocompleteComponent } from '@shared/components/entity/entity-key-autocomplete.component'; export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) { return markedOptionsService; @@ -425,14 +426,15 @@ export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) UploadImageDialogComponent, ImageDialogComponent, ImageReferencesComponent, - ImagesInUseDialogComponent, + ResourcesInUseDialogComponent, GalleryImageInputComponent, MultipleGalleryImageInputComponent, EmbedImageDialogComponent, ImageGalleryDialogComponent, WidgetButtonComponent, HexInputComponent, - ScadaSymbolInputComponent + ScadaSymbolInputComponent, + EntityKeyAutocompleteComponent, ], imports: [ CommonModule, @@ -688,13 +690,14 @@ export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) UploadImageDialogComponent, ImageDialogComponent, ImageReferencesComponent, - ImagesInUseDialogComponent, + ResourcesInUseDialogComponent, GalleryImageInputComponent, MultipleGalleryImageInputComponent, EmbedImageDialogComponent, ImageGalleryDialogComponent, WidgetButtonComponent, - ScadaSymbolInputComponent + ScadaSymbolInputComponent, + EntityKeyAutocompleteComponent, ] }) export class SharedModule { } diff --git a/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md b/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md new file mode 100644 index 0000000000..f8173dc528 --- /dev/null +++ b/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md @@ -0,0 +1 @@ + diff --git a/ui-ngx/src/assets/help/en_US/rulenode/save_timeseries_node_advanced.md b/ui-ngx/src/assets/help/en_US/rulenode/save_timeseries_node_advanced.md index 44a037f180..18c0f60cdb 100644 --- a/ui-ngx/src/assets/help/en_US/rulenode/save_timeseries_node_advanced.md +++ b/ui-ngx/src/assets/help/en_US/rulenode/save_timeseries_node_advanced.md @@ -2,23 +2,28 @@ When configuring the processing strategies, certain combinations can lead to unexpected behavior. Consider the following scenarios: +- **Skipping database storage** + + Choosing to disable one or more persistence actions (for instance, skipping database storage for Time series or Latest values while keeping WS updates enabled) introduces the risk of having only partial data available: + - If a message is processed only for real-time notifications (WebSockets) and not stored in the database, historical queries may not match data on the dashboard. + - When processing strategies for Time series and Latest values are out-of-sync, telemetry data may be stored in one table (e.g., Time series) while the same data is absent in the other (e.g., Latest values). + - **Disabling WebSocket (WS) updates** If WS updates are disabled, any changes to the time series data won’t be pushed to dashboards (or other WS subscriptions). This means that even if a database is updated, dashboards may not display the updated data until browser page is reloaded. +- **Skipping calculated field recalculation** + + If telemetry data is saved to the database while bypassing calculated field recalculation, the aggregated value may not update to reflect the latest data. + Conversely, if the calculated field is recalculated with new data but the corresponding telemetry value is not persisted in the database, the calculated field's value might include data that isn’t stored. + - **Different deduplication intervals across actions** When you configure different deduplication intervals for actions, the same incoming message might be processed differently for each action. For example, a message might be stored immediately in the Time series table (if set to *On every message*) while not being stored in the Latest values table because its deduplication interval hasn’t elapsed. Also, if the WebSocket updates are configured with a different interval, dashboards might show updates that do not match what is stored in the database. -- **Skipping database storage** - - Choosing to disable one or more persistence actions (for instance, skipping database storage for Time series or Latest values while keeping WS updates enabled) introduces the risk of having only partial data available: - - If a message is processed only for real-time notifications (WebSockets) and not stored in the database, historical queries may not match data on the dashboard. - - When processing strategies for Time series and Latest values are out-of-sync, telemetry data may be stored in one table (e.g., Time series) while the same data is absent in the other (e.g., Latest values). - - **Deduplication cache clearing** The deduplication mechanism uses a cache to track processed messages within each interval. diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 18801bda2c..02c604d8ae 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -65,6 +65,7 @@ "next-with-label": "Next: {{label}}", "read-more": "Read more", "hide": "Hide", + "test": "Test", "done": "Done", "print": "Print", "restore": "Restore", @@ -996,6 +997,7 @@ "failures": "Failures", "entity": "entity", "rule-node": "rule node", + "calculated-field": "calculated field", "hint": { "main": "All node debug messages rate limited with:", "main-limited": "All {{entity}} debug messages will be rate-limited, with a maximum of {{msg}} messages allowed per {{time}}.", @@ -1003,6 +1005,74 @@ "all-messages": "Save all debug events during time limit." } }, + "calculated-fields": { + "expression": "Expression", + "no-found": "No calculated fields found", + "list": "{ count, plural, =1 {One calculated field} other {List of # calculated fields} }", + "selected-fields": "{ count, plural, =1 {1 calculated field} other {# calculated fields} } selected", + "type": { + "simple": "Simple", + "script": "Script" + }, + "arguments": "Arguments", + "debugging": "Calculated field debugging", + "argument-name": "Argument name", + "datasource": "Datasource", + "add-argument": "Add argument", + "test-script-function": "Test script function", + "no-arguments": "No arguments configured", + "argument-settings": "Argument settings", + "argument-current": "Current entity", + "argument-current-tenant": "Current tenant", + "argument-device": "Device", + "argument-asset": "Asset", + "argument-customer": "Customer", + "argument-tenant": "Current tenant", + "argument-type": "Argument type", + "see-debug-events": "See debug events", + "attribute": "Attribute", + "timeseries-key": "Time series key", + "device-name": "Device name", + "latest-telemetry": "Latest telemetry", + "rolling": "Time series rolling", + "attribute-scope": "Attribute scope", + "server-attributes": "Server attributes", + "client-attributes": "Client attributes", + "shared-attributes": "Shared attributes", + "attribute-key": "Attribute key", + "default-value": "Default value", + "limit": "Max values", + "time-window": "Time window", + "customer-name": "Customer name", + "asset-name": "Asset name", + "timeseries": "Time series", + "output": "Output", + "create": "Create new calculated field", + "file": "Calculated field file", + "invalid-file-error": "Invalid file format. Please make sure the file is a valid JSON file.", + "import": "Import calculated field", + "export": "Export calculated field", + "export-failed-error": "Unable to export calculated field: {{error}}", + "output-type": "Output type", + "delete-title": "Are you sure you want to delete the calculated field '{{title}}'?", + "delete-text": "Be careful, after the confirmation the calculated field and all related data will become unrecoverable.", + "delete-multiple-title": "Are you sure you want to delete { count, plural, =1 {1 calculated field} other {# calculated fields} }?", + "delete-multiple-text": "Be careful, after the confirmation all selected calculated fields will be removed and all related data will become unrecoverable.", + "hint": { + "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with time series rolling type.", + "arguments-empty": "Arguments should not be empty.", + "expression-required": "Expression is required.", + "expression-invalid": "Expression is invalid", + "expression-max-length": "Expression length should be less than 255 characters.", + "argument-name-required": "Argument name is required.", + "argument-name-pattern": "Argument name is invalid.", + "argument-name-duplicate": "Argument with such name already exists.", + "argument-name-max-length": "Argument name should be less than 256 characters.", + "argument-type-required": "Argument type is required.", + "max-args": "Maximum number of arguments reached.", + "expression": "Default expression demonstrates how to transform a temperature from Fahrenheit to Celsius." + } + }, "confirm-on-exit": { "message": "You have unsaved changes. Are you sure you want to leave this page?", "html-message": "You have unsaved changes.
    Are you sure you want to leave this page?", @@ -1027,8 +1097,13 @@ "city-max-length": "Specified city should be less than 256" }, "common": { + "name": "Name", + "type": "Type", + "general": "General", "username": "Username", "password": "Password", + "data": "Data", + "timestamp": "Timestamp", "enter-username": "Enter username", "enter-password": "Enter password", "enter-search": "Enter search", @@ -1038,8 +1113,27 @@ "proceed": "Proceed", "open-details-page": "Open details page", "not-found": "Not found", + "value": "Value", "documentation": "Documentation", - "time-left": "{{time}} left" + "time-left": "{{time}} left", + "output": "Output", + "test-function": "Test function", + "test-with-this-message": "{{test}} with this message", + "suffix": { + "s": "s", + "ms": "ms" + }, + "hint": { + "name-required": "Name is required.", + "name-pattern": "Name is invalid.", + "name-max-length": "Name should be less than 256 characters.", + "title-required": "Title is required.", + "title-pattern": "Title is invalid.", + "title-max-length": "Title should be less than 256 characters.", + "key-required": "Key is required.", + "key-pattern": "Key is invalid.", + "key-max-length": "Key should be less than 256 characters." + } }, "content-type": { "json": "Json", @@ -2427,6 +2521,8 @@ "type-current-tenant": "Current Tenant", "type-current-user": "Current User", "type-current-user-owner": "Current User Owner", + "type-calculated-field": "Calculated Field", + "type-calculated-fields": "Calculated Fields", "type-widgets-bundle": "Widgets bundle", "type-widgets-bundles": "Widgets bundles", "list-of-widgets-bundles": "{ count, plural, =1 {One widgets bundle} other {List of # widget bundles} }", @@ -2627,6 +2723,9 @@ "type-stats": "Statistics", "type-debug-rule-node": "Debug", "type-debug-rule-chain": "Debug", + "type-debug-calculated-field": "Debug", + "arguments": "Arguments", + "result": "Result", "no-events-prompt": "No events found", "error": "Error", "alarm": "Alarm", @@ -4265,6 +4364,7 @@ "delete-javascript-resources-action-title": "Delete JavaScript { count, plural, =1 {1 resource} other {# resources} }", "delete-javascript-resources-text": "Please note that the selected JavaScript resources, even if they are used in JavaScript functions, will be deleted.", "delete-javascript-resources-title": "Are you sure you want to delete JavaScript { count, plural, =1 {1 resource} other {# resources} }?", + "delete-javascript-resource-in-use-text": "If you still want to delete the JavaScript resource, click the Delete anyway button.", "download": "Download JavaScript resource", "upload-from-file": "Upload JavaScript from file", "resource-file": "JavaScript resource file", @@ -4273,6 +4373,10 @@ "javascript-library": "JavaScript library", "javascript-type": "JavaScript type", "javascript-resource-details": "JavaScript resource details", + "javascript-resource-is-in-use": "JavaScript resource is used by other entities", + "javascript-resources-are-in-use": "JavaScript resources are used by other entities", + "javascript-resource-is-in-use-text": "The JavaScript resource '{{title}}' was not deleted because it is used by the following entities:", + "javascript-resources-are-in-use-text": "Not all JavaScript resources have been deleted because they are used by other entities.
    You can view referenced entities by clicking the References button in the corresponding resource row.
    If you still want to delete these JavaScript resources, select them in the table below and click the Delete selected button.", "search": "Search JavaScript resources", "selected-javascript-resources": "{ count, plural, =1 {1 JavaScript resource} other {# JavaScript resources} } selected", "no-javascript-resource-text": "No JavaScript resources found", @@ -5153,7 +5257,8 @@ }, "time-series": "Time series", "latest": "Latest values", - "web-sockets": "WebSockets" + "web-sockets": "WebSockets", + "calculated-fields": "Calculated fields" }, "key-val": { "key": "Key", @@ -5376,6 +5481,7 @@ "entities": "Entities", "rule-engine": "Rule Engine", "time-to-live": "Time-to-live", + "calculated-fields": "Calculated fields", "alarms-and-notifications": "Alarms and notifications", "ota-files-in-bytes": "Files", "ws-title": "WS", @@ -5428,6 +5534,21 @@ "tenant-entity-import-rate-limit": "Entity version load", "tenant-notification-request-rate-limit": "Notification requests", "tenant-notification-requests-per-rule-rate-limit": "Notification requests per notification rule", + "max-calculated-fields": "Maximum number of calculated fields per entity", + "max-calculated-fields-range": "Maximum number of calculated fields per entity can't be negative", + "max-calculated-fields-required": "Maximum number of calculated fields per entity is required", + "max-data-points-per-rolling-arg": "Maximum number of data points in a time series rolling arguments", + "max-data-points-per-rolling-arg-range": "Maximum number of data points in a time series rolling arguments can't be negative", + "max-data-points-per-rolling-arg-required": "Maximum number of data points in a time series rolling arguments is required", + "max-arguments-per-cf": "Maximum number of arguments per calculated field", + "max-arguments-per-cf-range": "Maximum number of arguments per calculated field can't be negative", + "max-arguments-per-cf-required": "Maximum number of arguments per calculated field is required", + "max-state-size": "Maximum size of the state in KB", + "max-state-size-range": "Maximum size of the state in KB can't be negative", + "max-state-size-required": "Maximum size of the state in KB is required", + "max-value-argument-size": "Maximum size of the single value argument in KB", + "max-value-argument-size-range": "Maximum size of the single value argument in KB can't be negative", + "max-value-argument-size-required": "Maximum size of the single value argument in KB is required", "max-transport-messages": "Transport messages maximum number", "max-transport-messages-required": "Transport messages maximum number is required.", "max-transport-messages-range": "Transport messages maximum number can't be negative", @@ -5499,6 +5620,8 @@ "advanced-settings": "Advanced settings", "edit-limit": "Edit limit", "but-less-than": "but less than", + "calculated-field-debug-event-rate-limit": "Calculated field debug events", + "edit-calculated-field-debug-event-rate-limit": "Edit calculated field debug events rate limits", "edit-transport-tenant-msg-title": "Edit transport tenant messages rate limits", "edit-transport-tenant-telemetry-msg-title": "Edit transport tenant telemetry messages rate limits", "edit-transport-tenant-telemetry-data-points-title": "Edit transport tenant telemetry data points rate limits", diff --git a/ui-ngx/src/form.scss b/ui-ngx/src/form.scss index e42fac19c3..8f3c7f0cd0 100644 --- a/ui-ngx/src/form.scss +++ b/ui-ngx/src/form.scss @@ -163,6 +163,13 @@ .tb-form-panel-title { font-weight: 500; font-size: 16px; + + &.tb-required::after { + font-size: 13px; + color: rgba(0, 0, 0, .54); + vertical-align: top; + content: " *"; + } } .tb-form-panel-hint { font-size: 12px;