diff --git a/.github/workflows/check-configuration-files.yml b/.github/workflows/check-configuration-files.yml index f280e326c2..561b7d0019 100644 --- a/.github/workflows/check-configuration-files.yml +++ b/.github/workflows/check-configuration-files.yml @@ -32,14 +32,14 @@ on: jobs: build: name: Check thingsboard.yml file - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Checkout code uses: actions/checkout@v2 - - name: Set up Python 3.10 + - name: Set up Python 3.13 uses: actions/setup-python@v3 with: - python-version: "3.10.2" + python-version: "3.13.2" architecture: "x64" env: AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache diff --git a/application/pom.xml b/application/pom.xml index a9356cc14a..6ce77646f8 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -124,6 +124,10 @@ org.thingsboard.common edge-api + + org.thingsboard.common + edqs + org.thingsboard dao @@ -369,6 +373,10 @@ com.google.firebase firebase-admin + + org.rocksdb + rocksdbjni + diff --git a/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json b/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json index 6701b59e0e..81f9e6a14d 100644 --- a/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json +++ b/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json @@ -50,8 +50,11 @@ }, "type": "org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode", "name": "Save Client Attributes", - "configurationVersion": 2, + "configurationVersion": 3, "configuration": { + "processingSettings": { + "type": "ON_EVERY_MESSAGE" + }, "scope": "CLIENT_SCOPE", "notifyDevice": false, "sendAttributesUpdatedNotification": false, @@ -119,7 +122,7 @@ "type": "org.thingsboard.rule.engine.edge.TbMsgPushToCloudNode", "name": "Push to cloud", "configuration": { - "scope": "SERVER_SCOPE" + "scope": "CLIENT_SCOPE" }, "externalId": null }, diff --git a/application/src/main/data/json/tenant/device_profile/rule_chain_template.json b/application/src/main/data/json/tenant/device_profile/rule_chain_template.json index 0f2473cde6..305dc04961 100644 --- a/application/src/main/data/json/tenant/device_profile/rule_chain_template.json +++ b/application/src/main/data/json/tenant/device_profile/rule_chain_template.json @@ -35,8 +35,11 @@ }, "type": "org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode", "name": "Save Client Attributes", - "configurationVersion": 2, + "configurationVersion": 3, "configuration": { + "processingSettings": { + "type": "ON_EVERY_MESSAGE" + }, "scope": "CLIENT_SCOPE", "notifyDevice": false, "sendAttributesUpdatedNotification": false, diff --git a/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json b/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json index 8efda98c5b..a988c9d5eb 100644 --- a/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json +++ b/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json @@ -34,8 +34,11 @@ }, "type": "org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode", "name": "Save Client Attributes", - "configurationVersion": 2, + "configurationVersion": 3, "configuration": { + "processingSettings": { + "type": "ON_EVERY_MESSAGE" + }, "scope": "CLIENT_SCOPE", "notifyDevice": false, "sendAttributesUpdatedNotification": false, diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index 2929635949..29c7a084f4 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -16,50 +16,70 @@ -- UPDATE SAVE TIME SERIES NODES START -DO $$ - BEGIN - -- Check if the rule_node table exists - IF EXISTS ( - SELECT 1 - FROM information_schema.tables - WHERE table_name = 'rule_node' - ) THEN +UPDATE rule_node +SET configuration = ( + (configuration::jsonb - 'skipLatestPersistence') + || jsonb_build_object( + 'processingSettings', jsonb_build_object( + 'type', 'ADVANCED', + 'timeseries', jsonb_build_object('type', 'ON_EVERY_MESSAGE'), + 'latest', jsonb_build_object('type', 'SKIP'), + 'webSockets', jsonb_build_object('type', 'ON_EVERY_MESSAGE'), + 'calculatedFields', jsonb_build_object('type', 'ON_EVERY_MESSAGE') + ) + ) + )::text, + configuration_version = 1 +WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode' + AND configuration_version = 0 + AND configuration::jsonb ->> 'skipLatestPersistence' = 'true'; - UPDATE rule_node - SET configuration = ( - (configuration::jsonb - 'skipLatestPersistence') - || jsonb_build_object( - 'processingSettings', jsonb_build_object( - 'type', 'ADVANCED', - 'timeseries', jsonb_build_object('type', 'ON_EVERY_MESSAGE'), - 'latest', jsonb_build_object('type', 'SKIP'), - 'webSockets', jsonb_build_object('type', 'ON_EVERY_MESSAGE') - ) - ) - )::text, - configuration_version = 1 - WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode' - AND configuration_version = 0 - AND configuration::jsonb ->> 'skipLatestPersistence' = 'true'; +UPDATE rule_node +SET configuration = ( + (configuration::jsonb - 'skipLatestPersistence') + || jsonb_build_object( + 'processingSettings', jsonb_build_object( + 'type', 'ON_EVERY_MESSAGE' + ) + ) + )::text, + configuration_version = 1 +WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode' + AND configuration_version = 0 + AND (configuration::jsonb ->> 'skipLatestPersistence' != 'true' OR configuration::jsonb ->> 'skipLatestPersistence' IS NULL); - UPDATE rule_node - SET configuration = ( - (configuration::jsonb - 'skipLatestPersistence') - || jsonb_build_object( - 'processingSettings', jsonb_build_object( - 'type', 'ON_EVERY_MESSAGE' - ) - ) - )::text, - configuration_version = 1 - WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode' - AND configuration_version = 0 - AND (configuration::jsonb ->> 'skipLatestPersistence' != 'true' OR configuration::jsonb ->> 'skipLatestPersistence' IS NULL); +-- UPDATE SAVE TIME SERIES NODES END - END IF; - END; -$$; +-- UPDATE SAVE ATTRIBUTES NODES START --- UPDATE SAVE TIME SERIES NODES END +UPDATE rule_node +SET configuration = ( + configuration::jsonb + || jsonb_build_object( + 'processingSettings', jsonb_build_object('type', 'ON_EVERY_MESSAGE') + ) + )::text, + configuration_version = 3 +WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode' + AND configuration_version = 2; + +-- UPDATE SAVE ATTRIBUTES NODES END + +ALTER TABLE api_usage_state ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; + +-- UPDATE TENANT PROFILE CALCULATED FIELD LIMITS START + +UPDATE tenant_profile +SET profile_data = profile_data + || jsonb_build_object( + 'configuration', profile_data->'configuration' || jsonb_build_object( + 'maxCalculatedFieldsPerEntity', COALESCE(profile_data->'configuration'->>'maxCalculatedFieldsPerEntity', '5')::bigint, + 'maxArgumentsPerCF', COALESCE(profile_data->'configuration'->>'maxArgumentsPerCF', '10')::bigint, + 'maxDataPointsPerRollingArg', COALESCE(profile_data->'configuration'->>'maxDataPointsPerRollingArg', '1000')::bigint, + 'maxStateSizeInKBytes', COALESCE(profile_data->'configuration'->>'maxStateSizeInKBytes', '32')::bigint, + 'maxSingleValueArgumentSizeInKBytes', COALESCE(profile_data->'configuration'->>'maxSingleValueArgumentSizeInKBytes', '2')::bigint + ) + ) +WHERE profile_data->'configuration'->>'maxCalculatedFieldsPerEntity' IS NULL; -ALTER TABLE api_usage_state ADD COLUMN IF NOT EXISTS version BIGINT DEFAULT 1; \ No newline at end of file +-- UPDATE TENANT PROFILE CALCULATED FIELD LIMITS END diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index eb76473386..2c22941c2f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -33,7 +33,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.NotificationCenter; -import org.thingsboard.rule.engine.api.RuleEngineDeviceStateManager; +import org.thingsboard.rule.engine.api.DeviceStateManager; import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.rule.engine.api.notification.SlackService; import org.thingsboard.rule.engine.api.sms.SmsSenderFactory; @@ -41,13 +41,18 @@ import org.thingsboard.script.api.js.JsInvokeService; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; +import org.thingsboard.server.cache.limits.RateLimitService; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.common.data.event.RuleChainDebugEvent; import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.limit.LimitedApi; +import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; @@ -62,6 +67,7 @@ import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.ClaimDevicesService; @@ -94,6 +100,7 @@ import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TenantProfileService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; @@ -101,6 +108,11 @@ import org.thingsboard.server.queue.discovery.DiscoveryService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; +import org.thingsboard.server.service.cf.CalculatedFieldQueueService; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; @@ -121,13 +133,17 @@ import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import org.thingsboard.server.service.transport.TbCoreToTransportService; +import org.thingsboard.server.utils.DebugModeRateLimitsConfig; import java.io.PrintWriter; import java.io.StringWriter; +import java.util.Map; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; @Slf4j @Component @@ -156,6 +172,18 @@ public class ActorSystemContext { } }; + private static final FutureCallback CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK = new FutureCallback<>() { + @Override + public void onSuccess(@Nullable Void event) { + + } + + @Override + public void onFailure(Throwable th) { + log.error("Could not save debug Event for Calculated Field", th); + } + }; + private final ConcurrentMap debugPerTenantLimits = new ConcurrentHashMap<>(); public ConcurrentMap getDebugPerTenantLimits() { @@ -206,7 +234,7 @@ public class ActorSystemContext { @Autowired(required = false) @Getter - private RuleEngineDeviceStateManager deviceStateManager; + private DeviceStateManager deviceStateManager; @Autowired @Getter @@ -289,6 +317,7 @@ public class ActorSystemContext { @Getter private TbEntityViewService tbEntityViewService; + @Lazy @Autowired @Getter private TelemetrySubscriptionService tsSubService; @@ -394,6 +423,10 @@ public class ActorSystemContext { @Getter private SlackService slackService; + @Autowired + @Getter + private CalculatedFieldService calculatedFieldService; + @Lazy @Autowired(required = false) @Getter @@ -416,6 +449,21 @@ public class ActorSystemContext { @Getter private TbCoreToTransportService tbCoreToTransportService; + @Lazy + @Autowired(required = false) + @Getter + private ApiLimitService apiLimitService; + + @Lazy + @Autowired(required = false) + @Getter + private RateLimitService rateLimitService; + + @Lazy + @Autowired(required = false) + @Getter + private DebugModeRateLimitsConfig debugModeRateLimitsConfig; + /** * The following Service will be null if we operate in tb-core mode */ @@ -487,6 +535,26 @@ public class ActorSystemContext { @Getter private EntityService entityService; + @Lazy + @Autowired(required = false) + @Getter + private CalculatedFieldProcessingService calculatedFieldProcessingService; + + @Lazy + @Autowired(required = false) + @Getter + private CalculatedFieldStateService calculatedFieldStateService; + + @Lazy + @Autowired(required = false) + @Getter + private CalculatedFieldQueueService calculatedFieldQueueService; + + @Lazy + @Autowired(required = false) + @Getter + private CalculatedFieldEntityProfileCache calculatedFieldEntityProfileCache; + @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter private long maxConcurrentSessionsPerDevice; @@ -558,14 +626,6 @@ public class ActorSystemContext { @Getter private long sessionReportTimeout; - @Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.enabled:true}") - @Getter - private boolean debugPerTenantEnabled; - - @Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") - @Getter - private String debugPerTenantLimitsConfiguration; - @Value("${actors.rpc.submit_strategy:BURST}") @Getter private String rpcSubmitStrategy; @@ -590,6 +650,10 @@ public class ActorSystemContext { @Getter private String deviceStateNodeRateLimitConfig; + @Value("${actors.calculated_fields.calculation_timeout:5}") + @Getter + private long cfCalculationResultTimeout; + @Getter @Setter private TbActorSystem actorSystem; @@ -719,9 +783,9 @@ public class ActorSystemContext { } private boolean checkLimits(TenantId tenantId, TbMsg tbMsg, Throwable error) { - if (debugPerTenantEnabled) { + if (debugModeRateLimitsConfig.isRuleChainDebugPerTenantLimitsEnabled()) { DebugTbRateLimits debugTbRateLimits = debugPerTenantLimits.computeIfAbsent(tenantId, id -> - new DebugTbRateLimits(new TbRateLimits(debugPerTenantLimitsConfiguration), false)); + new DebugTbRateLimits(new TbRateLimits(debugModeRateLimitsConfig.getRuleChainDebugPerTenantLimitsConfiguration()), false)); if (!debugTbRateLimits.getTbRateLimits().tryConsume()) { if (!debugTbRateLimits.isRuleChainEventSaved()) { @@ -751,6 +815,51 @@ public class ActorSystemContext { Futures.addCallback(future, RULE_CHAIN_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); } + public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, UUID tbMsgId, TbMsgType tbMsgType, String result, String errorMessage) { + if (checkLimits(tenantId)) { + try { + CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() + .tenantId(tenantId) + .entityId(calculatedFieldId.getId()) + .serviceId(getServiceId()) + .calculatedFieldId(calculatedFieldId) + .eventEntity(entityId); + if (tbMsgId != null) { + eventBuilder.msgId(tbMsgId); + } + if (tbMsgType != null) { + eventBuilder.msgType(tbMsgType.name()); + } + if (arguments != null) { + eventBuilder.arguments(JacksonUtil.toString( + arguments.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toTbelCfArg())) + )); + } + if (result != null) { + eventBuilder.result(result); + } + if (errorMessage != null) { + eventBuilder.error(errorMessage); + } + + ListenableFuture future = eventService.saveAsync(eventBuilder.build()); + Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); + } catch (IllegalArgumentException ex) { + log.warn("Failed to persist calculated field debug message", ex); + } + } + } + + private boolean checkLimits(TenantId tenantId) { + if (debugModeRateLimitsConfig.isCalculatedFieldDebugPerTenantLimitsEnabled() && + !rateLimitService.checkRateLimit(LimitedApi.CALCULATED_FIELD_DEBUG_EVENTS, (Object) tenantId, debugModeRateLimitsConfig.getCalculatedFieldDebugPerTenantLimitsConfiguration())) { + log.trace("[{}] Calculated field debug event limits exceeded!", tenantId); + return false; + } + return true; + } + public static Exception toException(Throwable error) { return Exception.class.isInstance(error) ? (Exception) error : new Exception(error); } diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index dc6a3bcf5e..50fa7e2f8d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; @@ -87,6 +88,7 @@ public class AppActor extends ContextAwareActor { case APP_INIT_MSG: break; case PARTITION_CHANGE_MSG: + case CF_PARTITIONS_CHANGE_MSG: ctx.broadcastToChildren(msg, true); break; case COMPONENT_LIFE_CYCLE_MSG: @@ -111,6 +113,18 @@ public class AppActor extends ContextAwareActor { case SESSION_TIMEOUT_MSG: ctx.broadcastToChildrenByType(msg, EntityType.TENANT); break; + case CF_INIT_MSG: + case CF_LINK_INIT_MSG: + case CF_STATE_RESTORE_MSG: + case CF_ENTITY_LIFECYCLE_MSG: + //TODO: use priority from the message body. For example, messages about CF lifecycle are important and Device lifecycle are not. + // same for the Linked telemetry. + onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, true); + break; + case CF_TELEMETRY_MSG: + case CF_LINKED_TELEMETRY_MSG: + onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, false); + break; default: return false; } @@ -175,6 +189,19 @@ public class AppActor extends ContextAwareActor { } } + private void onToCalculatedFieldSystemActorMsg(ToCalculatedFieldSystemMsg msg, boolean priority) { + getOrCreateTenantActor(msg.getTenantId()).ifPresentOrElse(tenantActor -> { + if (priority) { + tenantActor.tellWithHighPriority(msg); + } else { + tenantActor.tell(msg); + } + }, () -> { + msg.getCallback().onSuccess(); + }); + } + + private void onToDeviceActorMsg(TenantAwareMsg msg, boolean priority) { getOrCreateTenantActor(msg.getTenantId()).ifPresentOrElse(tenantActor -> { if (priority) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/AbstractCalculatedFieldActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/AbstractCalculatedFieldActor.java new file mode 100644 index 0000000000..6cf34599de --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/AbstractCalculatedFieldActor.java @@ -0,0 +1,70 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.DebugModeUtil; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.service.ContextAwareActor; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; + +@Slf4j +public abstract class AbstractCalculatedFieldActor extends ContextAwareActor { + + protected final TenantId tenantId; + + public AbstractCalculatedFieldActor(ActorSystemContext systemContext, TenantId tenantId) { + super(systemContext); + this.tenantId = tenantId; + } + + @Override + protected boolean doProcess(TbActorMsg msg) { + if (msg instanceof ToCalculatedFieldSystemMsg cfm) { + Exception cause; + try { + return doProcessCfMsg(cfm); + } catch (CalculatedFieldException cfe) { + if (DebugModeUtil.isDebugFailuresAvailable(cfe.getCtx().getCalculatedField())) { + String message; + if (cfe.getErrorMessage() != null) { + message = cfe.getErrorMessage(); + } else if (cfe.getCause() != null) { + message = cfe.getCause().getMessage(); + } else { + message = "N/A"; + } + systemContext.persistCalculatedFieldDebugEvent(tenantId, cfe.getCtx().getCfId(), cfe.getEventEntity(), cfe.getArguments(), cfe.getMsgId(), cfe.getMsgType(), null, message); + } + cause = cfe.getCause(); + } catch (Exception e) { + logProcessingException(e); + cause = e; + } + cfm.getCallback().onFailure(cause); + return true; + } else { + return false; + } + } + + abstract void logProcessingException(Exception e); + + abstract boolean doProcessCfMsg(ToCalculatedFieldSystemMsg msg) throws CalculatedFieldException; + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java new file mode 100644 index 0000000000..350a5776cf --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -0,0 +1,81 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActorCtx; +import org.thingsboard.server.actors.TbActorException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; + +@Slf4j +public class CalculatedFieldEntityActor extends AbstractCalculatedFieldActor { + + private final CalculatedFieldEntityMessageProcessor processor; + + CalculatedFieldEntityActor(ActorSystemContext systemContext, TenantId tenantId, EntityId entityId) { + super(systemContext, tenantId); + this.processor = new CalculatedFieldEntityMessageProcessor(systemContext, tenantId, entityId); + } + + @Override + public void init(TbActorCtx ctx) throws TbActorException { + super.init(ctx); + log.debug("[{}][{}] Starting CF entity actor.", processor.tenantId, processor.entityId); + try { + processor.init(ctx); + log.debug("[{}][{}] CF entity actor started.", processor.tenantId, processor.entityId); + } catch (Exception e) { + log.warn("[{}][{}] Unknown failure", processor.tenantId, processor.entityId, e); + throw new TbActorException("Failed to initialize CF entity actor", e); + } + } + + @Override + protected boolean doProcessCfMsg(ToCalculatedFieldSystemMsg msg) throws CalculatedFieldException { + switch (msg.getMsgType()) { + case CF_PARTITIONS_CHANGE_MSG: + processor.process((CalculatedFieldPartitionChangeMsg) msg); + break; + case CF_STATE_RESTORE_MSG: + processor.process((CalculatedFieldStateRestoreMsg) msg); + break; + case CF_ENTITY_INIT_CF_MSG: + processor.process((EntityInitCalculatedFieldMsg) msg); + break; + case CF_ENTITY_DELETE_MSG: + processor.process((CalculatedFieldEntityDeleteMsg) msg); + break; + case CF_ENTITY_TELEMETRY_MSG: + processor.process((EntityCalculatedFieldTelemetryMsg) msg); + break; + case CF_LINKED_TELEMETRY_MSG: + processor.process((EntityCalculatedFieldLinkedTelemetryMsg) msg); + break; + default: + return false; + } + return true; + } + + @Override + void logProcessingException(Exception e) { + log.warn("[{}][{}] Processing failure", tenantId, processor.entityId, e); + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java new file mode 100644 index 0000000000..6dc2f26050 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActor; +import org.thingsboard.server.actors.TbActorId; +import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; +import org.thingsboard.server.actors.TbEntityActorId; +import org.thingsboard.server.actors.device.DeviceActor; +import org.thingsboard.server.actors.service.ContextBasedCreator; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +public class CalculatedFieldEntityActorCreator extends ContextBasedCreator { + + private final TenantId tenantId; + private final EntityId entityId; + + public CalculatedFieldEntityActorCreator(ActorSystemContext context, TenantId tenantId, EntityId entityId) { + super(context); + this.tenantId = tenantId; + this.entityId = entityId; + } + + @Override + public TbActorId createActorId() { + return new TbCalculatedFieldEntityActorId(entityId); + } + + @Override + public TbActor createActor() { + return new CalculatedFieldEntityActor(context, tenantId, entityId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityDeleteMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityDeleteMsg.java new file mode 100644 index 0000000000..3ca6e8596a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityDeleteMsg.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; + +@Data +public class CalculatedFieldEntityDeleteMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final TbCallback callback; + + public CalculatedFieldEntityDeleteMsg(TenantId tenantId, + EntityId entityId, + TbCallback callback) { + this.tenantId = tenantId; + this.entityId = entityId; + this.callback = callback; + } + + @Override + public MsgType getMsgType() { + return MsgType.CF_ENTITY_DELETE_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java new file mode 100644 index 0000000000..9ab019097b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -0,0 +1,444 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import com.google.common.util.concurrent.ListenableFuture; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.DebugModeUtil; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActorCtx; +import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; +import org.thingsboard.server.service.cf.CalculatedFieldResult; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + + +/** + * @author Andrew Shvayka + */ +@Slf4j +public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareMsgProcessor { + // (1 for result persistence + 1 for the state persistence ) + public static final int CALLBACKS_PER_CF = 2; + + final TenantId tenantId; + final EntityId entityId; + final CalculatedFieldProcessingService cfService; + final CalculatedFieldStateService cfStateService; + final int partition; + + TbActorCtx ctx; + Map states = new HashMap<>(); + + CalculatedFieldEntityMessageProcessor(ActorSystemContext systemContext, TenantId tenantId, EntityId entityId) { + super(systemContext); + this.tenantId = tenantId; + this.entityId = entityId; + this.cfService = systemContext.getCalculatedFieldProcessingService(); + this.cfStateService = systemContext.getCalculatedFieldStateService(); + this.partition = systemContext.getCalculatedFieldEntityProfileCache().getEntityIdPartition(tenantId, entityId); + } + + void init(TbActorCtx ctx) { + this.ctx = ctx; + } + + public void process(CalculatedFieldPartitionChangeMsg msg) { + if (!msg.getPartitions()[partition]) { + log.info("[{}][{}] Stopping entity actor due to change partition event.", partition, entityId); + ctx.stop(ctx.getSelf()); + } + } + + public void process(CalculatedFieldStateRestoreMsg msg) { + CalculatedFieldId cfId = msg.getId().cfId(); + log.info("[{}] [{}] Processing CF state restore msg.", msg.getId().entityId(), cfId); + if (msg.getState() != null) { + states.put(cfId, msg.getState()); + } else { + states.remove(cfId); + } + } + + public void process(EntityInitCalculatedFieldMsg msg) throws CalculatedFieldException { + log.info("[{}] Processing entity init CF msg.", msg.getCtx().getCfId()); + var ctx = msg.getCtx(); + if (msg.isForceReinit()) { + log.info("Force reinitialization of CF: [{}].", ctx.getCfId()); + states.remove(ctx.getCfId()); + } + try { + var state = getOrInitState(ctx); + if (state.isSizeOk()) { + processStateIfReady(ctx, Collections.singletonList(ctx.getCfId()), state, null, null, msg.getCallback()); + } else { + throw new RuntimeException(ctx.getSizeExceedsLimitMessage()); + } + } catch (Exception e) { + if (e instanceof CalculatedFieldException cfe) { + throw cfe; + } + throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); + } + } + + public void process(CalculatedFieldEntityDeleteMsg msg) { + log.info("[{}] Processing CF entity delete msg.", msg.getEntityId()); + if (this.entityId.equals(msg.getEntityId())) { + if (states.isEmpty()) { + msg.getCallback().onSuccess(); + } else { + MultipleTbCallback multipleTbCallback = new MultipleTbCallback(states.size(), msg.getCallback()); + states.forEach((cfId, state) -> cfStateService.removeState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), multipleTbCallback)); + ctx.stop(ctx.getSelf()); + } + } else { + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + var state = states.remove(cfId); + if (state != null) { + cfStateService.removeState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); + } else { + msg.getCallback().onSuccess(); + } + } + } + + public void process(EntityCalculatedFieldTelemetryMsg msg) throws CalculatedFieldException { + log.info("[{}] Processing CF telemetry msg.", msg.getEntityId()); + var proto = msg.getProto(); + var numberOfCallbacks = CALLBACKS_PER_CF * (msg.getEntityIdFields().size() + msg.getProfileIdFields().size()); + MultipleTbCallback callback = new MultipleTbCallback(numberOfCallbacks, msg.getCallback()); + List cfIdList = getCalculatedFieldIds(proto); + Set cfIdSet = new HashSet<>(cfIdList); + for (var ctx : msg.getEntityIdFields()) { + process(ctx, proto, cfIdSet, cfIdList, callback); + } + for (var ctx : msg.getProfileIdFields()) { + process(ctx, proto, cfIdSet, cfIdList, callback); + } + } + + public void process(EntityCalculatedFieldLinkedTelemetryMsg msg) throws CalculatedFieldException { + log.info("[{}] Processing CF link telemetry msg.", msg.getEntityId()); + var proto = msg.getProto(); + var ctx = msg.getCtx(); + var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); + try { + List cfIds = getCalculatedFieldIds(proto); + if (cfIds.contains(ctx.getCfId())) { + callback.onSuccess(CALLBACKS_PER_CF); + } else { + if (proto.getTsDataCount() > 0) { + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); + } else if (proto.getAttrDataCount() > 0) { + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); + } else if (proto.getRemovedTsKeysCount() > 0) { + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArgumentsWithFetchedValue(ctx, proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); + } else if (proto.getRemovedAttrKeysCount() > 0) { + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArgumentsWithDefaultValue(ctx, msg.getEntityId(), proto.getScope(), proto.getRemovedAttrKeysList()), toTbMsgId(proto), toTbMsgType(proto)); + } else { + callback.onSuccess(CALLBACKS_PER_CF); + } + } + } catch (Exception e) { + throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); + } + } + + private void process(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, Collection cfIds, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { + try { + if (cfIds.contains(ctx.getCfId())) { + callback.onSuccess(CALLBACKS_PER_CF); + } else { + if (proto.getTsDataCount() > 0) { + processTelemetry(ctx, proto, cfIdList, callback); + } else if (proto.getAttrDataCount() > 0) { + processAttributes(ctx, proto, cfIdList, callback); + } else if (proto.getRemovedTsKeysCount() > 0) { + processRemovedTelemetry(ctx, proto, cfIdList, callback); + } else if (proto.getRemovedAttrKeysCount() > 0) { + processRemovedAttributes(ctx, proto, cfIdList, callback); + } else { + callback.onSuccess(CALLBACKS_PER_CF); + } + } + } catch (Exception e) { + if (e instanceof CalculatedFieldException cfe) { + throw cfe; + } + throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).cause(e).build(); + } + } + + private void processTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); + } + + private void processAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); + } + + private void processRemovedTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArgumentsWithFetchedValue(ctx, proto.getRemovedTsKeysList()), toTbMsgId(proto), toTbMsgType(proto)); + } + + private void processRemovedAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) throws CalculatedFieldException { + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArgumentsWithDefaultValue(ctx, proto.getScope(), proto.getRemovedAttrKeysList()), toTbMsgId(proto), toTbMsgType(proto)); + } + + private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List cfIdList, MultipleTbCallback callback, + Map newArgValues, UUID tbMsgId, TbMsgType tbMsgType) throws CalculatedFieldException { + if (newArgValues.isEmpty()) { + log.info("[{}] No new argument values to process for CF.", ctx.getCfId()); + callback.onSuccess(CALLBACKS_PER_CF); + } + CalculatedFieldState state = states.get(ctx.getCfId()); + boolean justRestored = false; + if (state == null) { + state = getOrInitState(ctx); + justRestored = true; + } + if (state.isSizeOk()) { + if (state.updateState(ctx, newArgValues) || justRestored) { + cfIdList = new ArrayList<>(cfIdList); + cfIdList.add(ctx.getCfId()); + processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback); + } else { + callback.onSuccess(CALLBACKS_PER_CF); + } + } else { + throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(); + } + } + + @SneakyThrows + private CalculatedFieldState getOrInitState(CalculatedFieldCtx ctx) { + CalculatedFieldState state = states.get(ctx.getCfId()); + if (state != null) { + return state; + } else { + ListenableFuture stateFuture = systemContext.getCalculatedFieldProcessingService().fetchStateFromDb(ctx, entityId); + // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. + // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. + // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, + // but this will significantly complicate the code. + state = stateFuture.get(1, TimeUnit.MINUTES); + state.checkStateSize(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), ctx.getMaxStateSize()); + states.put(ctx.getCfId(), state); + } + return state; + } + + private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) throws CalculatedFieldException { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); + boolean stateSizeChecked = false; + try { + if (ctx.isInitialized() && state.isReady()) { + CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(systemContext.getCfCalculationResultTimeout(), TimeUnit.SECONDS); + state.checkStateSize(ctxId, ctx.getMaxStateSize()); + stateSizeChecked = true; + if (state.isSizeOk()) { + cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); + if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null); + } + } + } + } catch (Exception e) { + throw CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).msgId(tbMsgId).msgType(tbMsgType).arguments(state.getArguments()).cause(e).build(); + } finally { + if (!stateSizeChecked) { + state.checkStateSize(ctxId, ctx.getMaxStateSize()); + } + if (state.isSizeOk()) { + cfStateService.persistState(ctxId, state, callback); + } else { + removeStateAndRaiseSizeException(ctxId, CalculatedFieldException.builder().ctx(ctx).eventEntity(entityId).errorMessage(ctx.getSizeExceedsLimitMessage()).build(), callback); + } + } + } + + private void removeStateAndRaiseSizeException(CalculatedFieldEntityCtxId ctxId, CalculatedFieldException ex, TbCallback callback) throws CalculatedFieldException { + // We remove the state, but remember that it is over-sized in a local map. + cfStateService.removeState(ctxId, new TbCallback() { + @Override + public void onSuccess() { + callback.onFailure(ex); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(ex); + } + }); + throw ex; + } + + private Map mapToArguments(CalculatedFieldCtx ctx, List data) { + return mapToArguments(ctx.getMainEntityArguments(), data); + } + + private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List data) { + var argNames = ctx.getLinkedEntityArguments().get(entityId); + if (argNames.isEmpty()) { + return Collections.emptyMap(); + } + return mapToArguments(argNames, data); + } + + private Map mapToArguments(Map argNames, List data) { + if (argNames.isEmpty()) { + return Collections.emptyMap(); + } + Map arguments = new HashMap<>(); + for (TsKvProto item : data) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); + String argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } + key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_ROLLING, null); + argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } + } + return arguments; + } + + private Map mapToArguments(CalculatedFieldCtx ctx, AttributeScopeProto scope, List attrDataList) { + return mapToArguments(ctx.getMainEntityArguments(), scope, attrDataList); + } + + private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List attrDataList) { + var argNames = ctx.getLinkedEntityArguments().get(entityId); + if (argNames.isEmpty()) { + return Collections.emptyMap(); + } + return mapToArguments(argNames, scope, attrDataList); + } + + private Map mapToArguments(Map argNames, AttributeScopeProto scope, List attrDataList) { + Map arguments = new HashMap<>(); + for (AttributeValueProto item : attrDataList) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); + String argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } + } + return arguments; + } + + private Map mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List removedAttrKeys) { + var argNames = ctx.getLinkedEntityArguments().get(entityId); + if (argNames.isEmpty()) { + return Collections.emptyMap(); + } + return mapToArgumentsWithDefaultValue(argNames, ctx.getArguments(), scope, removedAttrKeys); + } + + private Map mapToArgumentsWithDefaultValue(CalculatedFieldCtx ctx, AttributeScopeProto scope, List removedAttrKeys) { + return mapToArgumentsWithDefaultValue(ctx.getMainEntityArguments(), ctx.getArguments(), scope, removedAttrKeys); + } + + private Map mapToArgumentsWithDefaultValue(Map argNames, Map configArguments, AttributeScopeProto scope, List removedAttrKeys) { + Map arguments = new HashMap<>(); + for (String removedKey : removedAttrKeys) { + ReferencedEntityKey key = new ReferencedEntityKey(removedKey, ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); + String argName = argNames.get(key); + if (argName != null) { + Argument argument = configArguments.get(argName); + String defaultValue = (argument != null) ? argument.getDefaultValue() : null; + arguments.put(argName, StringUtils.isNotEmpty(defaultValue) + ? new SingleValueArgumentEntry(System.currentTimeMillis(), new StringDataEntry(removedKey, defaultValue), null) + : new SingleValueArgumentEntry()); + + } + } + return arguments; + } + + private Map mapToArgumentsWithFetchedValue(CalculatedFieldCtx ctx, List removedTelemetryKeys) { + Map deletedArguments = ctx.getArguments().entrySet().stream() + .filter(entry -> removedTelemetryKeys.contains(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + Map fetchedArgs = cfService.fetchArgsFromDb(tenantId, entityId, deletedArguments); + + fetchedArgs.values().forEach(arg -> arg.setForceResetPrevious(true)); + return fetchedArgs; + } + + private static List getCalculatedFieldIds(CalculatedFieldTelemetryMsgProto proto) { + List cfIds = new LinkedList<>(); + for (var cfId : proto.getPreviousCalculatedFieldsList()) { + cfIds.add(new CalculatedFieldId(new UUID(cfId.getCalculatedFieldIdMSB(), cfId.getCalculatedFieldIdLSB()))); + } + return cfIds; + } + + private UUID toTbMsgId(CalculatedFieldTelemetryMsgProto proto) { + if (proto.getTbMsgIdMSB() != 0 && proto.getTbMsgIdLSB() != 0) { + return new UUID(proto.getTbMsgIdMSB(), proto.getTbMsgIdLSB()); + } + return null; + } + + private TbMsgType toTbMsgType(CalculatedFieldTelemetryMsgProto proto) { + if (!proto.getTbMsgType().isEmpty()) { + return TbMsgType.valueOf(proto.getTbMsgType()); + } + return null; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldException.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldException.java new file mode 100644 index 0000000000..70c8dfbfd2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldException.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Builder; +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.Map; +import java.util.UUID; + +@Getter +@Builder +public class CalculatedFieldException extends Exception { + + private final CalculatedFieldCtx ctx; + private final EntityId eventEntity; + private final UUID msgId; + private final TbMsgType msgType; + private Map arguments; + private String errorMessage; + private Exception cause; + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldLinkedTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldLinkedTelemetryMsg.java new file mode 100644 index 0000000000..3e0fba2627 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldLinkedTelemetryMsg.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; + +@Data +public class CalculatedFieldLinkedTelemetryMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldLinkedTelemetryMsgProto proto; + private final TbCallback callback; + + + @Override + public MsgType getMsgType() { + return MsgType.CF_LINKED_TELEMETRY_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java new file mode 100644 index 0000000000..a5c935e83f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActorCtx; +import org.thingsboard.server.actors.TbActorException; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; + +/** + * Created by ashvayka on 15.03.18. + */ +@Slf4j +public class CalculatedFieldManagerActor extends AbstractCalculatedFieldActor { + + private final CalculatedFieldManagerMessageProcessor processor; + + public CalculatedFieldManagerActor(ActorSystemContext systemContext, TenantId tenantId) { + super(systemContext, tenantId); + this.processor = new CalculatedFieldManagerMessageProcessor(systemContext, tenantId); + } + + @Override + public void init(TbActorCtx ctx) throws TbActorException { + super.init(ctx); + log.debug("[{}] Starting CF manager actor.", processor.tenantId); + try { + processor.init(ctx); + log.debug("[{}] CF manager actor started.", processor.tenantId); + } catch (Exception e) { + log.warn("[{}] Unknown failure", processor.tenantId, e); + throw new TbActorException("Failed to initialize manager actor", e); + } + } + + @Override + protected boolean doProcessCfMsg(ToCalculatedFieldSystemMsg msg) throws CalculatedFieldException { + switch (msg.getMsgType()) { + case CF_PARTITIONS_CHANGE_MSG: + processor.onPartitionChange((CalculatedFieldPartitionChangeMsg) msg); + break; + case CF_INIT_MSG: + processor.onFieldInitMsg((CalculatedFieldInitMsg) msg); + break; + case CF_LINK_INIT_MSG: + processor.onLinkInitMsg((CalculatedFieldLinkInitMsg) msg); + break; + case CF_STATE_RESTORE_MSG: + processor.onStateRestoreMsg((CalculatedFieldStateRestoreMsg) msg); + break; + case CF_ENTITY_LIFECYCLE_MSG: + processor.onEntityLifecycleMsg((CalculatedFieldEntityLifecycleMsg) msg); + break; + case CF_TELEMETRY_MSG: + processor.onTelemetryMsg((CalculatedFieldTelemetryMsg) msg); + break; + case CF_LINKED_TELEMETRY_MSG: + processor.onLinkedTelemetryMsg((CalculatedFieldLinkedTelemetryMsg) msg); + break; + default: + return false; + } + return true; + } + + @Override + void logProcessingException(Exception e) { + log.warn("[{}] Processing failure", tenantId, e); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActorCreator.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActorCreator.java new file mode 100644 index 0000000000..99bf3cdbe9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActorCreator.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActor; +import org.thingsboard.server.actors.TbActorId; +import org.thingsboard.server.actors.TbEntityActorId; +import org.thingsboard.server.actors.TbStringActorId; +import org.thingsboard.server.actors.service.ContextBasedCreator; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +public class CalculatedFieldManagerActorCreator extends ContextBasedCreator { + + private final TenantId tenantId; + + public CalculatedFieldManagerActorCreator(ActorSystemContext context, TenantId tenantId) { + super(context); + this.tenantId = tenantId; + } + + @Override + public TbActorId createActorId() { + return new TbStringActorId("CFM|" + tenantId); + } + + @Override + public TbActor createActor() { + return new CalculatedFieldManagerActor(context, tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java new file mode 100644 index 0000000000..1418821081 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -0,0 +1,468 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActorCtx; +import org.thingsboard.server.actors.TbActorRef; +import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; +import org.thingsboard.server.actors.service.DefaultActorService; +import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; + +import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; + + +/** + * @author Andrew Shvayka + */ +@Slf4j +public class CalculatedFieldManagerMessageProcessor extends AbstractContextAwareMsgProcessor { + + private final Map calculatedFields = new HashMap<>(); + private final Map> entityIdCalculatedFields = new HashMap<>(); + private final Map> entityIdCalculatedFieldLinks = new HashMap<>(); + + private final CalculatedFieldProcessingService cfExecService; + private final CalculatedFieldStateService cfStateService; + private final CalculatedFieldEntityProfileCache cfEntityCache; + private final CalculatedFieldService cfDaoService; + private final TbAssetProfileCache assetProfileCache; + private final TbDeviceProfileCache deviceProfileCache; + protected final TenantId tenantId; + + protected TbActorCtx ctx; + + CalculatedFieldManagerMessageProcessor(ActorSystemContext systemContext, TenantId tenantId) { + super(systemContext); + this.cfEntityCache = systemContext.getCalculatedFieldEntityProfileCache(); + this.cfExecService = systemContext.getCalculatedFieldProcessingService(); + this.cfStateService = systemContext.getCalculatedFieldStateService(); + this.cfDaoService = systemContext.getCalculatedFieldService(); + this.assetProfileCache = systemContext.getAssetProfileCache(); + this.deviceProfileCache = systemContext.getDeviceProfileCache(); + this.tenantId = tenantId; + } + + void init(TbActorCtx ctx) { + this.ctx = ctx; + } + + public void onFieldInitMsg(CalculatedFieldInitMsg msg) throws CalculatedFieldException { + log.info("[{}] Processing CF init message.", msg.getCf().getId()); + var cf = msg.getCf(); + var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); + try { + cfCtx.init(); + } catch (Exception e) { + throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); + } + calculatedFields.put(cf.getId(), cfCtx); + // We use copy on write lists to safely pass the reference to another actor for the iteration. + // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); + msg.getCallback().onSuccess(); + } + + public void onLinkInitMsg(CalculatedFieldLinkInitMsg msg) { + log.info("[{}] Processing CF link init message for entity [{}].", msg.getLink().getCalculatedFieldId(), msg.getLink().getEntityId()); + var link = msg.getLink(); + // We use copy on write lists to safely pass the reference to another actor for the iteration. + // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) + entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link); + msg.getCallback().onSuccess(); + } + + public void onStateRestoreMsg(CalculatedFieldStateRestoreMsg msg) { + var cfId = msg.getId().cfId(); + var calculatedField = calculatedFields.get(cfId); + + if (calculatedField != null) { + msg.getState().setRequiredArguments(calculatedField.getArgNames()); + log.info("Pushing CF state restore msg to specific actor [{}]", msg.getId().entityId()); + getOrCreateActor(msg.getId().entityId()).tell(msg); + } else { + cfStateService.removeState(msg.getId(), msg.getCallback()); + } + } + + public void onEntityLifecycleMsg(CalculatedFieldEntityLifecycleMsg msg) throws CalculatedFieldException { + log.info("Processing entity lifecycle event: [{}] for entity: [{}]", msg.getData().getEvent(), msg.getData().getEntityId()); + var entityType = msg.getData().getEntityId().getEntityType(); + var event = msg.getData().getEvent(); + switch (entityType) { + case CALCULATED_FIELD: { + switch (event) { + case CREATED: + onCfCreated(msg.getData(), msg.getCallback()); + break; + case UPDATED: + onCfUpdated(msg.getData(), msg.getCallback()); + break; + case DELETED: + onCfDeleted(msg.getData(), msg.getCallback()); + break; + default: + msg.getCallback().onSuccess(); + break; + } + break; + } + case DEVICE: + case ASSET: { + switch (event) { + case CREATED: + onEntityCreated(msg.getData(), msg.getCallback()); + break; + case UPDATED: + onEntityUpdated(msg.getData(), msg.getCallback()); + break; + case DELETED: + onEntityDeleted(msg.getData(), msg.getCallback()); + break; + default: + msg.getCallback().onSuccess(); + break; + } + break; + } + default: { + msg.getCallback().onSuccess(); + } + } + } + + private void onEntityCreated(ComponentLifecycleMsg msg, TbCallback callback) { + EntityId entityId = msg.getEntityId(); + EntityId profileId = getProfileId(tenantId, entityId); + cfEntityCache.add(tenantId, profileId, entityId); + var entityIdFields = getCalculatedFieldsByEntityId(entityId); + var profileIdFields = getCalculatedFieldsByEntityId(profileId); + var fieldsCount = entityIdFields.size() + profileIdFields.size(); + if (fieldsCount > 0) { + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); + entityIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); + profileIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); + } else { + callback.onSuccess(); + } + } + + private void onEntityUpdated(ComponentLifecycleMsg msg, TbCallback callback) { + if (msg.getOldProfileId() != null && msg.getOldProfileId() != msg.getProfileId()) { + cfEntityCache.update(tenantId, msg.getOldProfileId(), msg.getProfileId(), msg.getEntityId()); + var oldProfileCfs = getCalculatedFieldsByEntityId(msg.getOldProfileId()); + var newProfileCfs = getCalculatedFieldsByEntityId(msg.getProfileId()); + var fieldsCount = oldProfileCfs.size() + newProfileCfs.size(); + if (fieldsCount > 0) { + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); + var entityId = msg.getEntityId(); + oldProfileCfs.forEach(ctx -> deleteCfForEntity(entityId, ctx.getCfId(), multiCallback)); + newProfileCfs.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); + } else { + callback.onSuccess(); + } + } + } + + private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { + cfEntityCache.evict(tenantId, msg.getEntityId()); + log.info("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); + getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); + } + + private void onCfCreated(ComponentLifecycleMsg msg, TbCallback callback) throws CalculatedFieldException { + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + if (calculatedFields.containsKey(cfId)) { + log.warn("[{}] CF was already initialized [{}]", tenantId, cfId); + callback.onSuccess(); + } else { + var cf = cfDaoService.findById(msg.getTenantId(), cfId); + if (cf == null) { + log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); + callback.onSuccess(); + } else { + var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); + try { + cfCtx.init(); + } catch (Exception e) { + throw CalculatedFieldException.builder().ctx(cfCtx).eventEntity(cf.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); + } + calculatedFields.put(cf.getId(), cfCtx); + // We use copy on write lists to safely pass the reference to another actor for the iteration. + // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); + addLinks(cf); + initCf(cfCtx, callback, false); + } + } + } + + private void onCfUpdated(ComponentLifecycleMsg msg, TbCallback callback) throws CalculatedFieldException { + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + var oldCfCtx = calculatedFields.get(cfId); + if (oldCfCtx == null) { + onCfCreated(msg, callback); + } else { + var newCf = cfDaoService.findById(msg.getTenantId(), cfId); + if (newCf == null) { + log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); + callback.onSuccess(); + } else { + var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); + try { + newCfCtx.init(); + } catch (Exception e) { + throw CalculatedFieldException.builder().ctx(newCfCtx).eventEntity(newCfCtx.getEntityId()).cause(e).errorMessage("Failed to initialize CF context").build(); + } + calculatedFields.put(newCf.getId(), newCfCtx); + List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); + List newCfList = new CopyOnWriteArrayList<>(); + boolean found = false; + for (CalculatedFieldCtx oldCtx : oldCfList) { + if (oldCtx.getCfId().equals(newCf.getId())) { + newCfList.add(newCfCtx); + found = true; + } else { + newCfList.add(oldCtx); + } + } + if (!found) { + newCfList.add(newCfCtx); + } + entityIdCalculatedFields.put(newCf.getEntityId(), newCfList); + + deleteLinks(oldCfCtx); + addLinks(newCf); + + // We use copy on write lists to safely pass the reference to another actor for the iteration. + // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) + var stateChanges = newCfCtx.hasStateChanges(oldCfCtx); + if (stateChanges || newCfCtx.hasOtherSignificantChanges(oldCfCtx)) { + initCf(newCfCtx, callback, stateChanges); + } else { + callback.onSuccess(); + } + } + } + } + + private void onCfDeleted(ComponentLifecycleMsg msg, TbCallback callback) { + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + var cfCtx = calculatedFields.remove(cfId); + if (cfCtx == null) { + log.warn("[{}] CF was already deleted [{}]", tenantId, cfId); + callback.onSuccess(); + } else { + entityIdCalculatedFields.get(cfCtx.getEntityId()).remove(cfCtx); + deleteLinks(cfCtx); + + EntityId entityId = cfCtx.getEntityId(); + EntityType entityType = cfCtx.getEntityId().getEntityType(); + if (isProfileEntity(entityType)) { + var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, entityId); + if (!entityIds.isEmpty()) { + //TODO: no need to do this if we cache all created actors and know which one belong to us; + var multiCallback = new MultipleTbCallback(entityIds.size(), callback); + entityIds.forEach(id -> deleteCfForEntity(id, cfId, multiCallback)); + } else { + callback.onSuccess(); + } + } else { + deleteCfForEntity(entityId, cfId, callback); + } + } + } + + public void onTelemetryMsg(CalculatedFieldTelemetryMsg msg) { + EntityId entityId = msg.getEntityId(); + log.info("Received telemetry msg from entity [{}]", entityId); + // 2 = 1 for CF processing + 1 for links processing + MultipleTbCallback callback = new MultipleTbCallback(2, msg.getCallback()); + // process all cfs related to entity, or it's profile; + var entityIdFields = getCalculatedFieldsByEntityId(entityId); + var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); + if (!entityIdFields.isEmpty() || !profileIdFields.isEmpty()) { + log.info("Pushing telemetry msg to specific actor [{}]", entityId); + getOrCreateActor(entityId).tell(new EntityCalculatedFieldTelemetryMsg(msg, entityIdFields, profileIdFields, callback)); + } else { + callback.onSuccess(); + } + // process all links (if any); + List linkedCalculatedFields = filterCalculatedFieldLinks(msg); + var linksSize = linkedCalculatedFields.size(); + if (linksSize > 0) { + cfExecService.pushMsgToLinks(msg, linkedCalculatedFields, callback); + } else { + callback.onSuccess(); + } + } + + public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsg msg) { + EntityId sourceEntityId = msg.getEntityId(); + log.info("Received linked telemetry msg from entity [{}]", sourceEntityId); + var proto = msg.getProto(); + var linksList = proto.getLinksList(); + for (var linkProto : linksList) { + var link = fromProto(linkProto); + var targetEntityId = link.entityId(); + var targetEntityType = targetEntityId.getEntityType(); + var cf = calculatedFields.get(link.cfId()); + if (EntityType.DEVICE_PROFILE.equals(targetEntityType) || EntityType.ASSET_PROFILE.equals(targetEntityType)) { + // iterate over all entities that belong to profile and push the message for corresponding CF + var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, targetEntityId); + if (!entityIds.isEmpty()) { + MultipleTbCallback callback = new MultipleTbCallback(entityIds.size(), msg.getCallback()); + var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, callback); + entityIds.forEach(entityId -> { + log.info("Pushing linked telemetry msg to specific actor [{}]", entityId); + getOrCreateActor(entityId).tell(newMsg); + }); + } else { + msg.getCallback().onSuccess(); + } + } else { + log.info("Pushing linked telemetry msg to specific actor [{}]", targetEntityId); + var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, msg.getCallback()); + getOrCreateActor(targetEntityId).tell(newMsg); + } + } + } + + private List filterCalculatedFieldLinks(CalculatedFieldTelemetryMsg msg) { + EntityId entityId = msg.getEntityId(); + var proto = msg.getProto(); + List result = new ArrayList<>(); + for (var link : getCalculatedFieldLinksByEntityId(entityId)) { + CalculatedFieldCtx ctx = calculatedFields.get(link.getCalculatedFieldId()); + if (ctx.linkMatches(entityId, proto)) { + result.add(ctx.toCalculatedFieldEntityCtxId()); + } + } + return result; + } + + private List getCalculatedFieldsByEntityId(EntityId entityId) { + if (entityId == null) { + return Collections.emptyList(); + } + var result = entityIdCalculatedFields.get(entityId); + if (result == null) { + result = Collections.emptyList(); + } + return result; + } + + private List getCalculatedFieldLinksByEntityId(EntityId entityId) { + if (entityId == null) { + return Collections.emptyList(); + } + var result = entityIdCalculatedFieldLinks.get(entityId); + if (result == null) { + result = Collections.emptyList(); + } + return result; + } + + private void initCf(CalculatedFieldCtx cfCtx, TbCallback callback, boolean forceStateReinit) { + EntityId entityId = cfCtx.getEntityId(); + EntityType entityType = cfCtx.getEntityId().getEntityType(); + if (isProfileEntity(entityType)) { + var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, entityId); + if (!entityIds.isEmpty()) { + var multiCallback = new MultipleTbCallback(entityIds.size(), callback); + entityIds.forEach(id -> initCfForEntity(id, cfCtx, forceStateReinit, multiCallback)); + } else { + callback.onSuccess(); + } + } else { + initCfForEntity(entityId, cfCtx, forceStateReinit, callback); + } + } + + private void deleteCfForEntity(EntityId entityId, CalculatedFieldId cfId, TbCallback callback) { + log.info("Pushing delete CF msg to specific actor [{}]", entityId); + getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, cfId, callback)); + } + + private void initCfForEntity(EntityId entityId, CalculatedFieldCtx cfCtx, boolean forceStateReinit, TbCallback callback) { + log.info("Pushing entity init CF msg to specific actor [{}]", entityId); + getOrCreateActor(entityId).tell(new EntityInitCalculatedFieldMsg(tenantId, cfCtx, callback, forceStateReinit)); + } + + private static boolean isProfileEntity(EntityType entityType) { + return EntityType.DEVICE_PROFILE.equals(entityType) || EntityType.ASSET_PROFILE.equals(entityType); + } + + private EntityId getProfileId(TenantId tenantId, EntityId entityId) { + return switch (entityId.getEntityType()) { + case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); + default -> null; + }; + } + + private TbActorRef getOrCreateActor(EntityId entityId) { + return ctx.getOrCreateChildActor(new TbCalculatedFieldEntityActorId(entityId), + () -> DefaultActorService.CF_ENTITY_DISPATCHER_NAME, + () -> new CalculatedFieldEntityActorCreator(systemContext, tenantId, entityId), + () -> true); + } + + private void addLinks(CalculatedField newCf) { + var newLinks = newCf.getConfiguration().buildCalculatedFieldLinks(tenantId, newCf.getEntityId(), newCf.getId()); + newLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link)); + } + + private void deleteLinks(CalculatedFieldCtx cfCtx) { + var oldCf = cfCtx.getCalculatedField(); + var oldLinks = oldCf.getConfiguration().buildCalculatedFieldLinks(tenantId, oldCf.getEntityId(), oldCf.getId()); + oldLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).remove(link)); + } + + public void onPartitionChange(CalculatedFieldPartitionChangeMsg msg) { + ctx.broadcastToChildren(msg, true); + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java new file mode 100644 index 0000000000..19be7c02fa --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; + +@Data +public class CalculatedFieldStateRestoreMsg implements ToCalculatedFieldSystemMsg { + + private final CalculatedFieldEntityCtxId id; + private final CalculatedFieldState state; + + @Override + public MsgType getMsgType() { + return MsgType.CF_STATE_RESTORE_MSG; + } + + @Override + public TenantId getTenantId() { + return id.tenantId(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldTelemetryMsg.java new file mode 100644 index 0000000000..68cd149cdf --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldTelemetryMsg.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; + +@Data +public class CalculatedFieldTelemetryMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldTelemetryMsgProto proto; + private final TbCallback callback; + + + @Override + public MsgType getMsgType() { + return MsgType.CF_TELEMETRY_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldLinkedTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldLinkedTelemetryMsg.java new file mode 100644 index 0000000000..b83aeae416 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldLinkedTelemetryMsg.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.List; + +@Data +public class EntityCalculatedFieldLinkedTelemetryMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldTelemetryMsgProto proto; + private final CalculatedFieldCtx ctx; + private final TbCallback callback; + + @Override + public MsgType getMsgType() { + return MsgType.CF_LINKED_TELEMETRY_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldTelemetryMsg.java new file mode 100644 index 0000000000..8ded4b6028 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldTelemetryMsg.java @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.List; + +@Data +public class EntityCalculatedFieldTelemetryMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldTelemetryMsgProto proto; + // Both lists are effectively immutable in CalculatedFieldManagerMessageProcessor and must stay so. + private final List entityIdFields; + private final List profileIdFields; + private final TbCallback callback; + + public EntityCalculatedFieldTelemetryMsg(CalculatedFieldTelemetryMsg msg, + List entityIdFields, + List profileIdFields, + TbCallback callback) { + this.tenantId = msg.getTenantId(); + this.entityId = msg.getEntityId(); + this.proto = msg.getProto(); + this.entityIdFields = entityIdFields; + this.profileIdFields = profileIdFields; + this.callback = callback; + } + + @Override + public MsgType getMsgType() { + return MsgType.CF_ENTITY_TELEMETRY_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java new file mode 100644 index 0000000000..1e8990ff8d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.List; + +@Data +public class EntityInitCalculatedFieldMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final CalculatedFieldCtx ctx; + private final TbCallback callback; + private final boolean forceReinit; + + @Override + public MsgType getMsgType() { + return MsgType.CF_ENTITY_INIT_CF_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java new file mode 100644 index 0000000000..d1f4c9092e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TbCallback; + +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +public class MultipleTbCallback implements TbCallback { + @Getter + private final UUID id; + private final AtomicInteger counter; + private final TbCallback callback; + + public MultipleTbCallback(int count, TbCallback callback) { + id = UUID.randomUUID(); + this.counter = new AtomicInteger(count); + this.callback = callback; + } + + @Override + public void onSuccess() { + onSuccess(1); + } + + public void onSuccess(int number) { + log.trace("[{}][{}] onSuccess({})", id, callback.getId(), number); + if (counter.addAndGet(-number) <= 0) { + log.trace("[{}][{}] Done.", id, callback.getId()); + callback.onSuccess(); + } + } + + @Override + public void onFailure(Throwable t) { + log.warn("[{}][{}] onFailure.", id, callback.getId()); + callback.onFailure(t); + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index cd8521027a..ee6758df6a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -28,8 +28,9 @@ import org.thingsboard.rule.engine.api.NotificationCenter; import org.thingsboard.rule.engine.api.RuleEngineAlarmService; import org.thingsboard.rule.engine.api.RuleEngineApiUsageStateService; import org.thingsboard.rule.engine.api.RuleEngineAssetProfileCache; +import org.thingsboard.rule.engine.api.RuleEngineCalculatedFieldQueueService; import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache; -import org.thingsboard.rule.engine.api.RuleEngineDeviceStateManager; +import org.thingsboard.rule.engine.api.DeviceStateManager; import org.thingsboard.rule.engine.api.RuleEngineRpcService; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.ScriptEngine; @@ -79,6 +80,7 @@ import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceCredentialsService; @@ -124,7 +126,6 @@ import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import java.util.function.Consumer; -import static org.thingsboard.server.common.data.DataConstants.MAIN_QUEUE_NAME; import static org.thingsboard.server.common.data.msg.TbMsgType.ATTRIBUTES_DELETED; import static org.thingsboard.server.common.data.msg.TbMsgType.ATTRIBUTES_UPDATED; import static org.thingsboard.server.common.data.msg.TbMsgType.ENTITY_CREATED; @@ -196,7 +197,7 @@ public class DefaultTbContext implements TbContext { @Override public void enqueue(TbMsg tbMsg, Runnable onSuccess, Consumer onFailure) { - enqueue(tbMsg, MAIN_QUEUE_NAME, onSuccess, onFailure); + enqueue(tbMsg, tbMsg.getQueueName(), onSuccess, onFailure); } @Override @@ -725,7 +726,7 @@ public class DefaultTbContext implements TbContext { } @Override - public RuleEngineDeviceStateManager getDeviceStateManager() { + public DeviceStateManager getDeviceStateManager() { return mainCtx.getDeviceStateManager(); } @@ -897,6 +898,16 @@ public class DefaultTbContext implements TbContext { return mainCtx.getSlackService(); } + @Override + public CalculatedFieldService getCalculatedFieldService() { + return mainCtx.getCalculatedFieldService(); + } + + @Override + public RuleEngineCalculatedFieldQueueService getCalculatedFieldQueueService() { + return mainCtx.getCalculatedFieldQueueService(); + } + @Override public boolean isExternalNodeForceAck() { return mainCtx.isExternalNodeForceAck(); diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index 6c8f253138..c2131d9e74 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -49,6 +49,8 @@ public class DefaultActorService extends TbApplicationEventListener deletedDevices; + private TbActorRef cfActor; private TenantActor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext, tenantId); @@ -95,6 +98,11 @@ public class TenantActor extends RuleChainManagerActor { } else { log.info("[{}] Skip init of the rule chains due to API limits", tenantId); } + //TODO: IM - extend API usage to have CF Exec Enabled? Not in 4.0; + cfActor = ctx.getOrCreateChildActor(new TbStringActorId("CFM|" + tenantId), + () -> DefaultActorService.CF_MANAGER_DISPATCHER_NAME, + () -> new CalculatedFieldManagerActorCreator(systemContext, tenantId), + () -> true); } catch (Exception e) { log.info("Failed to check ApiUsage \"ReExecEnabled\"!!!", e); cantFindTenant = true; @@ -159,12 +167,31 @@ public class TenantActor extends RuleChainManagerActor { case RULE_CHAIN_TO_RULE_CHAIN_MSG: onRuleChainMsg((RuleChainAwareMsg) msg); break; + case CF_INIT_MSG: + case CF_LINK_INIT_MSG: + case CF_STATE_RESTORE_MSG: + case CF_PARTITIONS_CHANGE_MSG: + case CF_ENTITY_LIFECYCLE_MSG: + onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, true); + break; + case CF_TELEMETRY_MSG: + case CF_LINKED_TELEMETRY_MSG: + onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, false); + break; default: return false; } return true; } + private void onToCalculatedFieldSystemActorMsg(ToCalculatedFieldSystemMsg msg, boolean priority) { + if (priority) { + cfActor.tellWithHighPriority(msg); + } else { + cfActor.tell(msg); + } + } + private boolean isMyPartition(EntityId entityId) { return systemContext.resolve(ServiceType.TB_CORE, tenantId, entityId).isMyPartition(); } diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index cf9d6444a2..73e278389a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -70,6 +70,7 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.domain.Domain; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeInfo; @@ -80,6 +81,7 @@ import org.thingsboard.server.common.data.id.AlarmCommentId; import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceId; @@ -132,6 +134,7 @@ import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.ClaimDevicesService; @@ -367,6 +370,9 @@ public abstract class BaseController { @Autowired protected NotificationTargetService notificationTargetService; + @Autowired + protected CalculatedFieldService calculatedFieldService; + @Value("${server.log_controller_error_stack_trace}") @Getter private boolean logControllerErrorStackTrace; @@ -672,6 +678,9 @@ public abstract class BaseController { case MOBILE_APP_BUNDLE: checkMobileAppBundleId(new MobileAppBundleId(entityId.getId()), operation); return; + case CALCULATED_FIELD: + checkCalculatedFieldId(new CalculatedFieldId(entityId.getId()), operation); + return; default: checkEntityId(entityId, entitiesService::findEntityByTenantIdAndId, operation); } @@ -955,6 +964,10 @@ public abstract class BaseController { } } + protected CalculatedField checkCalculatedFieldId(CalculatedFieldId calculatedFieldId, Operation operation) throws ThingsboardException { + return checkEntityId(calculatedFieldId, calculatedFieldService::findById, operation); + } + protected HomeDashboardInfo getHomeDashboardInfo(SecurityUser securityUser, JsonNode additionalInfo) { HomeDashboardInfo homeDashboardInfo = extractHomeDashboardInfoFromAdditionalInfo(additionalInfo); if (homeDashboardInfo == null) { @@ -982,7 +995,8 @@ public abstract class BaseController { } return new HomeDashboardInfo(dashboardId, hideDashboardToolbar); } - } catch (Exception ignored) {} + } catch (Exception ignored) { + } return null; } diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java new file mode 100644 index 0000000000..f899d0f480 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -0,0 +1,283 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.script.api.tbel.TbelCfCtx; +import org.thingsboard.script.api.tbel.TbelCfSingleValueArg; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.event.EventType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.config.annotations.ApiOperation; +import org.thingsboard.server.dao.event.EventService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldScriptEngine; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldTbelScriptEngine; +import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +import static org.thingsboard.server.controller.ControllerConstants.CF_TEXT_SEARCH_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.ENTITY_ID_PARAM_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.ENTITY_TYPE_PARAM_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_END; +import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_START; +import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.SORT_PROPERTY_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH; +import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; +import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; + +@RestController +@TbCoreComponent +@RequestMapping("/api") +@RequiredArgsConstructor +@Slf4j +public class CalculatedFieldController extends BaseController { + + private final TbCalculatedFieldService tbCalculatedFieldService; + private final EventService eventService; + private final TbelInvokeService tbelInvokeService; + + public static final String CALCULATED_FIELD_ID = "calculatedFieldId"; + + public static final int TIMEOUT = 20; + + private static final String TEST_SCRIPT_EXPRESSION = "Execute the Script expression and return the result. The format of request: \n\n" + + MARKDOWN_CODE_BLOCK_START + + "{\n" + + " \"expression\": \"var temp = 0; foreach(element: temperature.values) {temp += element.value;} var avgTemperature = temp / temperature.values.size(); var adjustedTemperature = avgTemperature + 0.1 * humidity.value; return {\\\"adjustedTemperature\\\": adjustedTemperature};\",\n" + + " \"arguments\": {\n" + + " \"temperature\": {\n" + + " \"type\": \"TS_ROLLING\",\n" + + " \"timeWindow\": {\n" + + " \"startTs\": 1739775630002,\n" + + " \"endTs\": 65432211,\n" + + " \"limit\": 5\n" + + " },\n" + + " \"values\": [\n" + + " { \"ts\": 1739775639851, \"value\": 23 },\n" + + " { \"ts\": 1739775664561, \"value\": 43 },\n" + + " { \"ts\": 1739775713079, \"value\": 15 },\n" + + " { \"ts\": 1739775999522, \"value\": 34 },\n" + + " { \"ts\": 1739776228452, \"value\": 22 }\n" + + " ]\n" + + " },\n" + + " \"humidity\": { \"type\": \"SINGLE_VALUE\", \"ts\": 1739776478057, \"value\": 23 }\n" + + " }\n" + + "}" + + MARKDOWN_CODE_BLOCK_END + + "\n\n Expected result JSON contains \"output\" and \"error\"."; + + @ApiOperation(value = "Create Or Update Calculated Field (saveCalculatedField)", + notes = "Creates or Updates the Calculated Field. When creating calculated field, platform generates Calculated Field Id as " + UUID_WIKI_LINK + + "The newly created Calculated Field Id will be present in the response. " + + "Specify existing Calculated Field Id to update the calculated field. " + + "Referencing non-existing Calculated Field Id will cause 'Not Found' error. " + + "Remove 'id', 'tenantId' from the request body example (below) to create new Calculated Field entity. " + + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/calculatedField", method = RequestMethod.POST) + @ResponseBody + public CalculatedField saveCalculatedField(@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "A JSON value representing the calculated field.") + @RequestBody CalculatedField calculatedField) throws Exception { + calculatedField.setTenantId(getTenantId()); + checkEntity(calculatedField.getId(), calculatedField, Resource.CALCULATED_FIELD); + checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); + checkReferencedEntities(calculatedField.getConfiguration(), getCurrentUser()); + return tbCalculatedFieldService.save(calculatedField, getCurrentUser()); + } + + @ApiOperation(value = "Get Calculated Field (getCalculatedFieldById)", + notes = "Fetch the Calculated Field object based on the provided Calculated Field Id." + ) + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/calculatedField/{calculatedFieldId}", method = RequestMethod.GET) + @ResponseBody + public CalculatedField getCalculatedFieldById(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { + checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); + CalculatedField calculatedField = tbCalculatedFieldService.findById(calculatedFieldId, getCurrentUser()); + checkNotNull(calculatedField); + checkEntityId(calculatedField.getEntityId(), Operation.READ_CALCULATED_FIELD); + return calculatedField; + } + + @ApiOperation(value = "Get Calculated Fields by Entity Id (getCalculatedFieldsByEntityId)", + notes = "Fetch the Calculated Fields based on the provided Entity Id." + ) + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/{entityType}/{entityId}/calculatedFields", params = {"pageSize", "page"}, method = RequestMethod.GET) + @ResponseBody + public PageData getCalculatedFieldsByEntityId( + @Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true, schema = @Schema(defaultValue = "DEVICE")) @PathVariable("entityType") String entityType, + @Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @PathVariable("entityId") String entityIdStr, + @Parameter(description = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, + @Parameter(description = PAGE_NUMBER_DESCRIPTION, required = true) @RequestParam int page, + @Parameter(description = CF_TEXT_SEARCH_DESCRIPTION) @RequestParam(required = false) String textSearch, + @Parameter(description = SORT_PROPERTY_DESCRIPTION, schema = @Schema(allowableValues = {"createdTime", "name"})) @RequestParam(required = false) String sortProperty, + @Parameter(description = SORT_ORDER_DESCRIPTION, schema = @Schema(allowableValues = {"ASC", "DESC"})) @RequestParam(required = false) String sortOrder) throws ThingsboardException { + PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); + checkParameter("entityId", entityIdStr); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityIdStr); + checkEntityId(entityId, Operation.READ_CALCULATED_FIELD); + return checkNotNull(tbCalculatedFieldService.findAllByTenantIdAndEntityId(entityId, getCurrentUser(), pageLink)); + } + + @ApiOperation(value = "Delete Calculated Field (deleteCalculatedField)", + notes = "Deletes the calculated field. Referencing non-existing Calculated Field Id will cause an error." + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/calculatedField/{calculatedFieldId}", method = RequestMethod.DELETE) + @ResponseStatus(value = HttpStatus.OK) + public void deleteCalculatedField(@PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws Exception { + checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); + CalculatedField calculatedField = checkCalculatedFieldId(calculatedFieldId, Operation.DELETE); + checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); + tbCalculatedFieldService.delete(calculatedField, getCurrentUser()); + } + + @ApiOperation(value = "Get latest calculated field debug event (getLatestCalculatedFieldDebugEvent)", + notes = "Gets latest calculated field debug event for specified calculated field id. " + + "Referencing non-existing calculated field id will cause an error. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/calculatedField/{calculatedFieldId}/debug", method = RequestMethod.GET) + @ResponseBody + public JsonNode getLatestCalculatedFieldDebugEvent(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { + checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); + CalculatedField calculatedField = checkCalculatedFieldId(calculatedFieldId, Operation.READ); + checkEntityId(calculatedField.getEntityId(), Operation.READ_CALCULATED_FIELD); + TenantId tenantId = getCurrentUser().getTenantId(); + return Optional.ofNullable(eventService.findLatestEvents(tenantId, calculatedFieldId, EventType.DEBUG_CALCULATED_FIELD, 1)) + .flatMap(events -> events.stream().map(EventInfo::getBody).findFirst()) + .orElse(null); + } + + @ApiOperation(value = "Test Script expression", + notes = TEST_SCRIPT_EXPRESSION + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/calculatedField/testScript", method = RequestMethod.POST) + @ResponseBody + public JsonNode testScript( + @io.swagger.v3.oas.annotations.parameters.RequestBody(description = "Test calculated field TBEL expression.") + @RequestBody JsonNode inputParams) { + String expression = inputParams.get("expression").asText(); + Map arguments = Objects.requireNonNullElse( + JacksonUtil.convertValue(inputParams.get("arguments"), new TypeReference<>() { + }), + Collections.emptyMap() + ); + + ArrayList ctxAndArgNames = new ArrayList<>(arguments.size() + 1); + ctxAndArgNames.add("ctx"); + ctxAndArgNames.addAll(arguments.keySet()); + + String output = ""; + String errorText = ""; + + try { + if (tbelInvokeService == null) { + throw new IllegalArgumentException("TBEL script engine is disabled!"); + } + + CalculatedFieldScriptEngine calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( + getTenantId(), + tbelInvokeService, + expression, + ctxAndArgNames.toArray(String[]::new) + ); + + + Object[] args = new Object[ctxAndArgNames.size()]; + args[0] = new TbelCfCtx(arguments); + for (int i = 1; i < ctxAndArgNames.size(); i++) { + var arg = arguments.get(ctxAndArgNames.get(i)); + if (arg instanceof TbelCfSingleValueArg svArg) { + args[i] = svArg.getValue(); + } else { + args[i] = arg; + } + } + + JsonNode json = calculatedFieldScriptEngine.executeJsonAsync(args).get(TIMEOUT, TimeUnit.SECONDS); + output = JacksonUtil.toString(json); + } catch (Exception e) { + log.error("Error evaluating expression", e); + errorText = e.getMessage(); + } + + ObjectNode result = JacksonUtil.newObjectNode(); + result.put("output", output); + result.put("error", errorText); + return result; + } + + private & HasTenantId, I extends EntityId> void checkReferencedEntities(CalculatedFieldConfiguration calculatedFieldConfig, SecurityUser user) throws ThingsboardException { + List referencedEntityIds = calculatedFieldConfig.getReferencedEntities(); + for (EntityId referencedEntityId : referencedEntityIds) { + EntityType entityType = referencedEntityId.getEntityType(); + switch (entityType) { + case TENANT, CUSTOMER, ASSET, DEVICE -> checkEntityId(referencedEntityId, Operation.READ); + default -> + throw new IllegalArgumentException("Calculated fields do not support '" + entityType + "' for referenced entities."); + } + } + + } + +} diff --git a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java index 4987e5c26f..e45d041950 100644 --- a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java +++ b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java @@ -96,6 +96,7 @@ public class ControllerConstants { protected static final String EDGE_TEXT_SEARCH_DESCRIPTION = "The case insensitive 'substring' filter based on the edge name."; protected static final String EVENT_TEXT_SEARCH_DESCRIPTION = "The value is not used in searching."; protected static final String AUDIT_LOG_TEXT_SEARCH_DESCRIPTION = "The case insensitive 'substring' filter based on one of the next properties: entityType, entityName, userName, actionType, actionStatus."; + protected static final String CF_TEXT_SEARCH_DESCRIPTION = "The case insensitive 'substring' filter based on the calculated field name."; protected static final String SORT_PROPERTY_DESCRIPTION = "Property of entity to sort by"; protected static final String SORT_ORDER_DESCRIPTION = "Sort order. ASC (ASCENDING) or DESC (DESCENDING)"; diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java index 0387e8c24a..7fd0b12077 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java @@ -20,6 +20,8 @@ import io.swagger.v3.oas.annotations.media.Schema; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; @@ -27,6 +29,7 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; @@ -38,6 +41,8 @@ import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.msg.edqs.EdqsApiService; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.query.EntityQueryService; @@ -55,6 +60,10 @@ public class EntityQueryController extends BaseController { @Autowired private EntityQueryService entityQueryService; + @Autowired + private EdqsService edqsService; + @Autowired + private EdqsApiService edqsApiService; private static final int MAX_PAGE_SIZE = 100; @@ -133,4 +142,16 @@ public class EntityQueryController extends BaseController { return entityQueryService.getKeysByQuery(getCurrentUser(), tenantId, query, isTimeseries, isAttributes, scope); } + @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") + @PostMapping("/edqs/system/request") + public void processSystemEdqsRequest(@RequestBody ToCoreEdqsRequest request) { + edqsService.processSystemRequest(request); + } + + @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") + @GetMapping("/edqs/enabled") + public boolean isEdqsApiEnabled() { + return edqsApiService.isEnabled(); + } + } diff --git a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java index bc22313a03..4ee86871d6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java +++ b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java @@ -35,8 +35,8 @@ import org.thingsboard.server.common.data.SystemParams; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.mobile.qrCodeSettings.QrCodeSettings; import org.thingsboard.server.common.data.mobile.qrCodeSettings.QRCodeConfig; +import org.thingsboard.server.common.data.mobile.qrCodeSettings.QrCodeSettings; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.settings.UserSettings; import org.thingsboard.server.common.data.settings.UserSettingsType; @@ -46,6 +46,7 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; +import org.thingsboard.server.utils.DebugModeRateLimitsConfig; import java.util.Collections; import java.util.List; @@ -74,12 +75,6 @@ public class SystemInfoController extends BaseController { @Value("${debug.settings.default_duration:15}") private int defaultDebugDurationMinutes; - @Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.enabled:true}") - private boolean ruleChainDebugPerTenantLimitsEnabled; - - @Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") - private String ruleChainDebugPerTenantLimitsConfiguration; - @Autowired(required = false) private BuildProperties buildProperties; @@ -89,6 +84,9 @@ public class SystemInfoController extends BaseController { @Autowired private QrCodeSettingService qrCodeSettingService; + @Autowired + private DebugModeRateLimitsConfig debugModeRateLimitsConfig; + @PostConstruct public void init() { JsonNode info = buildInfoObject(); @@ -152,9 +150,14 @@ public class SystemInfoController extends BaseController { DefaultTenantProfileConfiguration tenantProfileConfiguration = tenantProfileCache.get(tenantId).getDefaultProfileConfiguration(); systemParams.setMaxResourceSize(tenantProfileConfiguration.getMaxResourceSize()); systemParams.setMaxDebugModeDurationMinutes(DebugModeUtil.getMaxDebugAllDuration(tenantProfileConfiguration.getMaxDebugModeDurationMinutes(), defaultDebugDurationMinutes)); - if (ruleChainDebugPerTenantLimitsEnabled) { - systemParams.setRuleChainDebugPerTenantLimitsConfiguration(ruleChainDebugPerTenantLimitsConfiguration); + if (debugModeRateLimitsConfig.isRuleChainDebugPerTenantLimitsEnabled()) { + systemParams.setRuleChainDebugPerTenantLimitsConfiguration(debugModeRateLimitsConfig.getRuleChainDebugPerTenantLimitsConfiguration()); + } + if (debugModeRateLimitsConfig.isCalculatedFieldDebugPerTenantLimitsEnabled()) { + systemParams.setCalculatedFieldDebugPerTenantLimitsConfiguration(debugModeRateLimitsConfig.getCalculatedFieldDebugPerTenantLimitsConfiguration()); } + systemParams.setMaxArgumentsPerCF(tenantProfileConfiguration.getMaxArgumentsPerCF()); + systemParams.setMaxDataPointsPerRollingArg(tenantProfileConfiguration.getMaxDataPointsPerRollingArg()); } systemParams.setMobileQrEnabled(Optional.ofNullable(qrCodeSettingService.findQrCodeSettings(TenantId.SYS_TENANT_ID)) .map(QrCodeSettings::getQrCodeConfig).map(QRCodeConfig::isShowOnHomePage) diff --git a/application/src/main/java/org/thingsboard/server/controller/TbResourceController.java b/application/src/main/java/org/thingsboard/server/controller/TbResourceController.java index 9c72b9a5ad..b23603f6a2 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TbResourceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TbResourceController.java @@ -41,6 +41,7 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.ResourceSubType; import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TbResourceDeleteResult; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.TbResourceInfoFilter; import org.thingsboard.server.common.data.exception.ThingsboardException; @@ -322,12 +323,14 @@ public class TbResourceController extends BaseController { notes = "Deletes the Resource. Referencing non-existing Resource Id will cause an error." + SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')") @DeleteMapping(value = "/resource/{resourceId}") - public void deleteResource(@Parameter(description = RESOURCE_ID_PARAM_DESCRIPTION) - @PathVariable("resourceId") String strResourceId) throws ThingsboardException { + public ResponseEntity deleteResource(@Parameter(description = RESOURCE_ID_PARAM_DESCRIPTION) + @PathVariable("resourceId") String strResourceId, + @RequestParam(name = "force", required = false) boolean force) throws ThingsboardException { checkParameter(RESOURCE_ID, strResourceId); TbResourceId resourceId = new TbResourceId(toUUID(strResourceId)); TbResource tbResource = checkResourceId(resourceId, Operation.DELETE); - tbResourceService.delete(tbResource, getCurrentUser()); + TbResourceDeleteResult tbResourceDeleteResult = tbResourceService.delete(tbResource, force, getCurrentUser()); + return (tbResourceDeleteResult.isSuccess() ? ResponseEntity.ok() : ResponseEntity.badRequest()).body(tbResourceDeleteResult); } private ResponseEntity downloadResourceIfChanged(String strResourceId, String etag) throws ThingsboardException { diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java b/application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java index ff4de0ce0c..1a3bdce1f6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java @@ -18,6 +18,7 @@ package org.thingsboard.server.controller; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.ArraySchema; import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpStatus; @@ -160,7 +161,12 @@ public class TenantProfileController extends BaseController { " \"rpcTtlDays\": 0,\n" + " \"queueStatsTtlDays\": 0,\n" + " \"ruleEngineExceptionsTtlDays\": 0,\n" + - " \"warnThreshold\": 0\n" + + " \"warnThreshold\": 0,\n" + + " \"maxCalculatedFieldsPerEntity\": 5,\n" + + " \"maxArgumentsPerCF\": 10,\n" + + " \"maxDataPointsPerRollingArg\": 1000,\n" + + " \"maxStateSizeInKBytes\": 32,\n" + + " \"maxSingleValueArgumentSizeInKBytes\": 2" + " }\n" + " },\n" + " \"default\": false\n" + @@ -172,7 +178,7 @@ public class TenantProfileController extends BaseController { @RequestMapping(value = "/tenantProfile", method = RequestMethod.POST) @ResponseBody public TenantProfile saveTenantProfile(@Parameter(description = "A JSON value representing the tenant profile.") - @RequestBody TenantProfile tenantProfile) throws ThingsboardException { + @Valid @RequestBody TenantProfile tenantProfile) throws ThingsboardException { TenantProfile oldProfile; if (tenantProfile.getId() == null) { accessControlService.checkPermission(getCurrentUser(), Resource.TENANT_PROFILE, Operation.CREATE); diff --git a/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java b/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java new file mode 100644 index 0000000000..a30b7218a1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.exception; + +public class CalculatedFieldStateException extends RuntimeException { + + public CalculatedFieldStateException(String message) { + super(message); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java index aecb31ca0c..b0ab4edc27 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -15,13 +15,11 @@ */ package org.thingsboard.server.service.apiusage; -import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.ListenableFuture; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; @@ -92,15 +90,7 @@ import java.util.stream.Collectors; public class DefaultTbApiUsageStateService extends AbstractPartitionBasedService implements TbApiUsageStateService { public static final String HOURLY = "Hourly"; - public static final FutureCallback VOID_CALLBACK = new FutureCallback() { - @Override - public void onSuccess(@Nullable Void result) { - } - @Override - public void onFailure(Throwable t) { - } - }; private final PartitionService partitionService; private final TenantService tenantService; private final TimeseriesService tsService; @@ -219,7 +209,6 @@ public class DefaultTbApiUsageStateService extends AbstractPartitionBasedService .tenantId(tenantId) .entityId(usageState.getApiUsageState().getId()) .entries(updatedEntries) - .callback(VOID_CALLBACK) .build()); if (!result.isEmpty()) { persistAndNotify(usageState, result); @@ -331,7 +320,6 @@ public class DefaultTbApiUsageStateService extends AbstractPartitionBasedService .tenantId(tenantId) .entityId(id) .entries(profileThresholds) - .callback(VOID_CALLBACK) .build()); } } @@ -364,7 +352,6 @@ public class DefaultTbApiUsageStateService extends AbstractPartitionBasedService .tenantId(state.getTenantId()) .entityId(state.getApiUsageState().getId()) .entries(stateTelemetry) - .callback(VOID_CALLBACK) .build()); if (state.getEntityType() == EntityType.TENANT && !state.getEntityId().equals(TenantId.SYS_TENANT_ID)) { @@ -457,7 +444,6 @@ public class DefaultTbApiUsageStateService extends AbstractPartitionBasedService .tenantId(state.getTenantId()) .entityId(state.getApiUsageState().getId()) .entries(counts) - .callback(VOID_CALLBACK) .build()); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java new file mode 100644 index 0000000000..0163d65d98 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java @@ -0,0 +1,72 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.exception.CalculatedFieldStateException; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; + +import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; +import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; + +public abstract class AbstractCalculatedFieldStateService implements CalculatedFieldStateService { + + @Autowired + private ActorSystemContext actorSystemContext; + + protected PartitionedQueueConsumerManager> eventConsumer; + + @Override + public void init(PartitionedQueueConsumerManager> eventConsumer) { + this.eventConsumer = eventConsumer; + } + + @Override + public final void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + if (state.isSizeExceedsLimit()) { + throw new CalculatedFieldStateException("State size exceeds the maximum allowed limit. The state will not be persisted to RocksDB."); + } + doPersist(stateId, toProto(stateId, state), callback); + } + + protected abstract void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback); + + @Override + public final void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback) { + doRemove(stateId, callback); + } + + protected abstract void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback); + + protected void processRestoredState(CalculatedFieldStateProto stateMsg) { + var id = fromProto(stateMsg.getId()); + var state = fromProto(stateMsg); + processRestoredState(id, state); + } + + protected void processRestoredState(CalculatedFieldEntityCtxId id, CalculatedFieldState state) { + actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(id, state)); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java new file mode 100644 index 0000000000..fb63432fed --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.List; + +public interface CalculatedFieldCache { + + CalculatedField getCalculatedField(CalculatedFieldId calculatedFieldId); + + List getCalculatedFieldsByEntityId(EntityId entityId); + + List getCalculatedFieldLinksByEntityId(EntityId entityId); + + CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId); + + List getCalculatedFieldCtxsByEntityId(EntityId entityId); + + void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + void evict(CalculatedFieldId calculatedFieldId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldInitService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldInitService.java new file mode 100644 index 0000000000..6505dae581 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldInitService.java @@ -0,0 +1,19 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +public interface CalculatedFieldInitService { +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java new file mode 100644 index 0000000000..847caccaff --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; + +import java.util.List; +import java.util.Map; + +public interface CalculatedFieldProcessingService { + + ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); + + Map fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map arguments); + + void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculationResult, List cfIds, TbCallback callback); + + void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List linkedCalculatedFields, TbCallback callback); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java new file mode 100644 index 0000000000..eb86220361 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import com.google.common.util.concurrent.FutureCallback; +import org.thingsboard.rule.engine.api.AttributesDeleteRequest; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; +import org.thingsboard.rule.engine.api.RuleEngineCalculatedFieldQueueService; +import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; + +import java.util.List; + +public interface CalculatedFieldQueueService extends RuleEngineCalculatedFieldQueueService { + + /** + * Filter CFs based on the request entity. Push to the queue if any matching CF exist; + * + * @param request - telemetry save request; + * @param callback + */ + void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback); + + void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback); + + void pushRequestToQueue(AttributesDeleteRequest request, List result, FutureCallback callback); + + void pushRequestToQueue(TimeseriesDeleteRequest request, List result, FutureCallback callback); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java new file mode 100644 index 0000000000..8eb27395c1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.configuration.OutputType; + +@Data +public final class CalculatedFieldResult { + + private final OutputType type; + private final AttributeScope scope; + private final JsonNode result; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java new file mode 100644 index 0000000000..109f13f183 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.exception.CalculatedFieldStateException; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; + +import java.util.Set; + +public interface CalculatedFieldStateService { + + void init(PartitionedQueueConsumerManager> eventConsumer); + + void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) throws CalculatedFieldStateException; + + void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback); + + void restore(Set partitions); + + void stop(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java b/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java new file mode 100644 index 0000000000..f95227bc24 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import org.rocksdb.Options; +import org.rocksdb.WriteOptions; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.edqs.util.TbRocksDb; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='in-memory'") +public class CfRocksDb extends TbRocksDb { + + public CfRocksDb(@Value("${queue.calculated_fields.rocks_db_path:${user.home}/.rocksdb/cf_states}") String path) { + super(path, new Options().setCreateIfMissing(true), new WriteOptions().setSync(true)); + } + + @PostConstruct + @Override + public void init() { + super.init(); + } + + @PreDestroy + @Override + public void close() { + super.close(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java new file mode 100644 index 0000000000..64487d9b3e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -0,0 +1,187 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultCalculatedFieldCache implements CalculatedFieldCache { + + private static final Integer UNKNOWN_PARTITION = -1; + + private final Lock calculatedFieldFetchLock = new ReentrantLock(); + + private final CalculatedFieldService calculatedFieldService; + private final TbelInvokeService tbelInvokeService; + private final ActorSystemContext actorSystemContext; + private final ApiLimitService apiLimitService; + + private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> entityIdCalculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); + + @Value("${calculatedField.initFetchPackSize:50000}") + @Getter + private int initFetchPackSize; + + @AfterStartUp(order = AfterStartUp.CF_READ_CF_SERVICE) + public void init() { + //TODO: move to separate place to avoid circular references with the ActorSystemContext (@Lazy for tsSubService) + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + cfs.forEach(cf -> { + calculatedFields.putIfAbsent(cf.getId(), cf); + actorSystemContext.tell(new CalculatedFieldInitMsg(cf.getTenantId(), cf)); + }); + calculatedFields.values().forEach(cf -> { + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cf); + }); + PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); + cfls.forEach(link -> { + calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new CopyOnWriteArrayList<>()).add(link); + actorSystemContext.tell(new CalculatedFieldLinkInitMsg(link.getTenantId(), link)); + }); + calculatedFieldLinks.values().stream() + .flatMap(List::stream) + .forEach(link -> + entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link) + ); + } + + @Override + public CalculatedField getCalculatedField(CalculatedFieldId calculatedFieldId) { + return calculatedFields.get(calculatedFieldId); + } + + @Override + public List getCalculatedFieldsByEntityId(EntityId entityId) { + return entityIdCalculatedFields.getOrDefault(entityId, new CopyOnWriteArrayList<>()); + } + + @Override + public List getCalculatedFieldLinksByEntityId(EntityId entityId) { + return entityIdCalculatedFieldLinks.getOrDefault(entityId, new CopyOnWriteArrayList<>()); + } + + @Override + public CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId) { + CalculatedFieldCtx ctx = calculatedFieldsCtx.get(calculatedFieldId); + if (ctx == null) { + calculatedFieldFetchLock.lock(); + try { + ctx = calculatedFieldsCtx.get(calculatedFieldId); + if (ctx == null) { + CalculatedField calculatedField = getCalculatedField(calculatedFieldId); + if (calculatedField != null) { + ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService, apiLimitService); + calculatedFieldsCtx.put(calculatedFieldId, ctx); + log.debug("[{}] Put calculated field ctx into cache: {}", calculatedFieldId, ctx); + } + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found calculated field ctx in cache: {}", calculatedFieldId, ctx); + return ctx; + } + + @Override + public List getCalculatedFieldCtxsByEntityId(EntityId entityId) { + if (entityId == null) { + return Collections.emptyList(); + } + return getCalculatedFieldsByEntityId(entityId).stream() + .map(cf -> getCalculatedFieldCtx(cf.getId())) + .toList(); + } + + @Override + public void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + calculatedFieldFetchLock.lock(); + try { + CalculatedField calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); + EntityId cfEntityId = calculatedField.getEntityId(); + + calculatedFields.put(calculatedFieldId, calculatedField); + + entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new CopyOnWriteArrayList<>()).add(calculatedField); + + CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); + calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); + + configuration.getReferencedEntities().stream() + .filter(referencedEntityId -> !referencedEntityId.equals(cfEntityId)) + .forEach(referencedEntityId -> { + entityIdCalculatedFieldLinks.computeIfAbsent(referencedEntityId, entityId -> new CopyOnWriteArrayList<>()) + .add(configuration.buildCalculatedFieldLink(tenantId, referencedEntityId, calculatedFieldId)); + }); + } finally { + calculatedFieldFetchLock.unlock(); + } + } + + @Override + public void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + evict(calculatedFieldId); + addCalculatedField(tenantId, calculatedFieldId); + } + + @Override + public void evict(CalculatedFieldId calculatedFieldId) { + CalculatedField oldCalculatedField = calculatedFields.remove(calculatedFieldId); + log.debug("[{}] evict calculated field from cache: {}", calculatedFieldId, oldCalculatedField); + calculatedFieldLinks.remove(calculatedFieldId); + log.debug("[{}] evict calculated field from cached calculated fields by entity id: {}", calculatedFieldId, oldCalculatedField); + entityIdCalculatedFields.forEach((entityId, calculatedFields) -> calculatedFields.removeIf(cf -> cf.getId().equals(calculatedFieldId))); + log.debug("[{}] evict calculated field links from cache: {}", calculatedFieldId, oldCalculatedField); + calculatedFieldsCtx.remove(calculatedFieldId); + log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); + entityIdCalculatedFieldLinks.forEach((entityId, calculatedFieldLinks) -> calculatedFieldLinks.removeIf(link -> link.getCalculatedFieldId().equals(calculatedFieldId))); + log.debug("[{}] evict calculated field links from cached links by entity id: {}", calculatedFieldId, oldCalculatedField); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java new file mode 100644 index 0000000000..9442329edb --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; + +@Slf4j +@Service +@TbRuleEngineComponent +@RequiredArgsConstructor +public class DefaultCalculatedFieldInitService implements CalculatedFieldInitService { + + private final CalculatedFieldEntityProfileCache entityProfileCache; + private final AssetService assetService; + private final DeviceService deviceService; + + @Value("${calculated_fields.init_fetch_pack_size:50000}") + @Getter + private int initFetchPackSize; + + @AfterStartUp(order = AfterStartUp.CF_READ_PROFILE_ENTITIES_SERVICE) + public void initCalculatedFieldDefinitions() { + PageDataIterable deviceIdInfos = new PageDataIterable<>(deviceService::findProfileEntityIdInfos, initFetchPackSize); + for (ProfileEntityIdInfo idInfo : deviceIdInfos) { + log.trace("Processing device record: {}", idInfo); + entityProfileCache.add(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId()); + } + PageDataIterable assetIdInfos = new PageDataIterable<>(assetService::findProfileEntityIdInfos, initFetchPackSize); + for (ProfileEntityIdInfo idInfo : assetIdInfos) { + log.trace("Processing asset record: {}", idInfo); + entityProfileCache.add(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId()); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java new file mode 100644 index 0000000000..27bc0120c4 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -0,0 +1,325 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.math.NumberUtils; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; +import org.thingsboard.server.actors.calculatedField.MultipleTbCallback; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +import static org.thingsboard.server.common.data.DataConstants.SCOPE; +import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; + +@TbRuleEngineComponent +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultCalculatedFieldProcessingService implements CalculatedFieldProcessingService { + + private final AttributesService attributesService; + private final TimeseriesService timeseriesService; + private final TbClusterService clusterService; + private final ApiLimitService apiLimitService; + private final PartitionService partitionService; + + private ListeningExecutorService calculatedFieldCallbackExecutor; + + @PostConstruct + public void init() { + calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); + } + + @PreDestroy + public void stop() { + if (calculatedFieldCallbackExecutor != null) { + calculatedFieldCallbackExecutor.shutdownNow(); + } + } + + @Override + public ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId) { + Map> argFutures = new HashMap<>(); + for (var entry : ctx.getArguments().entrySet()) { + var argEntityId = entry.getValue().getRefEntityId() != null ? entry.getValue().getRefEntityId() : entityId; + var argValueFuture = fetchKvEntry(ctx.getTenantId(), argEntityId, entry.getValue()); + argFutures.put(entry.getKey(), argValueFuture); + } + return Futures.whenAllComplete(argFutures.values()).call(() -> { + var result = createStateByType(ctx); + result.updateState(ctx, argFutures.entrySet().stream() + .collect(Collectors.toMap( + Entry::getKey, // Keep the key as is + entry -> { + try { + // Resolve the future to get the value + return entry.getValue().get(); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException("Error getting future result for key: " + entry.getKey(), e); + } + } + ))); + return result; + }, calculatedFieldCallbackExecutor); + } + + @Override + public Map fetchArgsFromDb(TenantId tenantId, EntityId entityId, Map arguments) { + Map> argFutures = new HashMap<>(); + for (var entry : arguments.entrySet()) { + var argEntityId = entry.getValue().getRefEntityId() != null ? entry.getValue().getRefEntityId() : entityId; + var argValueFuture = fetchKvEntry(tenantId, argEntityId, entry.getValue()); + argFutures.put(entry.getKey(), argValueFuture); + } + return argFutures.entrySet().stream() + .collect(Collectors.toMap( + Entry::getKey, // Keep the key as is + entry -> { + try { + // Resolve the future to get the value + return entry.getValue().get(); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException("Error getting future result for key: " + entry.getKey(), e); + } + } + )); + } + + @Override + public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculatedFieldResult, List cfIds, TbCallback callback) { + try { + OutputType type = calculatedFieldResult.getType(); + TbMsgType msgType = OutputType.ATTRIBUTES.equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; + TbMsgMetaData md = OutputType.ATTRIBUTES.equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; + TbMsg msg = TbMsg.newMsg().type(msgType).originator(entityId).previousCalculatedFieldIds(cfIds).metaData(md).data(JacksonUtil.writeValueAsString(calculatedFieldResult.getResult())).build(); + clusterService.pushMsgToRuleEngine(tenantId, entityId, msg, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + callback.onSuccess(); + log.trace("[{}][{}] Pushed message to rule engine: {} ", tenantId, entityId, msg); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + }); + } catch (Exception e) { + log.warn("[{}][{}] Failed to push message to rule engine. CalculatedFieldResult: {}", tenantId, entityId, calculatedFieldResult, e); + callback.onFailure(e); + } + } + + @Override + public void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List linkedCalculatedFields, TbCallback callback) { + Map> unicasts = new HashMap<>(); + List broadcasts = new ArrayList<>(); + for (CalculatedFieldEntityCtxId link : linkedCalculatedFields) { + var linkEntityId = link.entityId(); + var linkEntityType = linkEntityId.getEntityType(); + // Let's assume number of entities in profile is N, and number of partitions is P. If N > P, we save by broadcasting to all partitions. Usually N >> P. + boolean broadcast = EntityType.DEVICE_PROFILE.equals(linkEntityType) || EntityType.ASSET_PROFILE.equals(linkEntityType); + if (broadcast) { + broadcasts.add(link); + } else { + TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF, link.entityId()); + unicasts.computeIfAbsent(tpi, k -> new ArrayList<>()).add(link); + } + } + MultipleTbCallback linkCallback = new MultipleTbCallback(2, callback); + if (!broadcasts.isEmpty()) { + broadcast(broadcasts, msg, linkCallback); + } else { + linkCallback.onSuccess(); + } + if (!unicasts.isEmpty()) { + unicast(unicasts, msg, linkCallback); + } else { + linkCallback.onSuccess(); + } + } + + private void unicast(Map> unicasts, CalculatedFieldTelemetryMsg msg, MultipleTbCallback mainCallback) { + TbQueueCallback callback = new TbCallbackWrapper(new MultipleTbCallback(unicasts.size(), mainCallback)); + unicasts.forEach((topicPartitionInfo, ctxIds) -> { + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(msg.getProto(), ctxIds); + clusterService.pushMsgToCalculatedFields(topicPartitionInfo, UUID.randomUUID(), + ToCalculatedFieldMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), callback); + }); + } + + private void broadcast(List broadcasts, CalculatedFieldTelemetryMsg msg, MultipleTbCallback mainCallback) { + TbQueueCallback callback = new TbCallbackWrapper(mainCallback); + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(msg.getProto(), broadcasts); + clusterService.broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), callback); + } + + private CalculatedFieldLinkedTelemetryMsgProto buildLinkedTelemetryMsgProto(CalculatedFieldTelemetryMsgProto telemetryProto, List links) { + Builder builder = CalculatedFieldLinkedTelemetryMsgProto.newBuilder(); + builder.setMsg(telemetryProto); + for (CalculatedFieldEntityCtxId link : links) { + builder.addLinks(toProto(link)); + } + return builder.build(); + } + + private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { + return switch (argument.getRefEntityKey().getType()) { + case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument); + case ATTRIBUTE -> transformSingleValueArgument( + Futures.transform( + attributesService.find(tenantId, entityId, argument.getRefEntityKey().getScope(), argument.getRefEntityKey().getKey()), + result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(createDefaultKvEntry(argument), System.currentTimeMillis(), 0L))), + calculatedFieldCallbackExecutor) + ); + case TS_LATEST -> transformSingleValueArgument( + Futures.transform( + timeseriesService.findLatest(tenantId, entityId, argument.getRefEntityKey().getKey()), + result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument), 0L))), + calculatedFieldCallbackExecutor)); + }; + } + + private ListenableFuture transformSingleValueArgument(ListenableFuture> kvEntryFuture) { + return Futures.transform(kvEntryFuture, kvEntry -> { + if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { + return ArgumentEntry.createSingleValueArgument(kvEntry.get()); + } else { + return new SingleValueArgumentEntry(); + } + }, calculatedFieldCallbackExecutor); + } + + private ListenableFuture fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument) { + long currentTime = System.currentTimeMillis(); + long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); + long startTs = currentTime - timeWindow; + long maxDataPoints = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); + int argumentLimit = argument.getLimit(); + int limit = argumentLimit == 0 || argumentLimit > maxDataPoints ? (int) maxDataPoints : argument.getLimit(); + + ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); + ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); + + return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? new TsRollingArgumentEntry(limit, timeWindow) : ArgumentEntry.createTsRollingArgument(tsRolling, limit, timeWindow), calculatedFieldCallbackExecutor); + } + + private KvEntry createDefaultKvEntry(Argument argument) { + String key = argument.getRefEntityKey().getKey(); + String defaultValue = argument.getDefaultValue(); + if (StringUtils.isBlank(defaultValue)) { + return new StringDataEntry(key, null); + } + if (NumberUtils.isParsable(defaultValue)) { + return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); + } + if ("true".equalsIgnoreCase(defaultValue) || "false".equalsIgnoreCase(defaultValue)) { + return new BooleanDataEntry(key, Boolean.parseBoolean(defaultValue)); + } + return new StringDataEntry(key, defaultValue); + } + + private CalculatedFieldState createStateByType(CalculatedFieldCtx ctx) { + return switch (ctx.getCfType()) { + case SIMPLE -> new SimpleCalculatedFieldState(ctx.getArgNames()); + case SCRIPT -> new ScriptCalculatedFieldState(ctx.getArgNames()); + }; + } + + private static class TbCallbackWrapper implements TbQueueCallback { + private final TbCallback callback; + + public TbCallbackWrapper(TbCallback callback) { + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + callback.onSuccess(); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java new file mode 100644 index 0000000000..ba4be6ace6 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -0,0 +1,275 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import com.google.common.util.concurrent.FutureCallback; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.rule.engine.api.AttributesDeleteRequest; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; +import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.function.Predicate; +import java.util.function.Supplier; + +import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; +import static org.thingsboard.server.utils.CalculatedFieldUtils.toProto; + +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueService { + + public static final TbQueueCallback DUMMY_TB_QUEUE_CALLBACK = new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + } + + @Override + public void onFailure(Throwable t) { + } + }; + + private final TbAssetProfileCache assetProfileCache; + private final TbDeviceProfileCache deviceProfileCache; + private final CalculatedFieldCache calculatedFieldCache; + private final TbClusterService clusterService; + + private static final Set supportedReferencedEntities = EnumSet.of( + EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT + ); + + @Override + public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback) { + var tenantId = request.getTenantId(); + var entityId = request.getEntityId(); + checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries()), cf -> cf.linkMatches(entityId, request.getEntries()), + () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); + } + + @Override + public void pushRequestToQueue(TimeseriesSaveRequest request, FutureCallback callback) { + pushRequestToQueue(request, null, callback); + } + + @Override + public void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback) { + var tenantId = request.getTenantId(); + var entityId = request.getEntityId(); + checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), + () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); + } + + @Override + public void pushRequestToQueue(AttributesSaveRequest request, FutureCallback callback) { + pushRequestToQueue(request, null, callback); + } + + @Override + public void pushRequestToQueue(AttributesDeleteRequest request, List result, FutureCallback callback) { + var tenantId = request.getTenantId(); + var entityId = request.getEntityId(); + checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matchesKeys(result, request.getScope()), cf -> cf.linkMatchesAttrKeys(entityId, result, request.getScope()), + () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); + } + + @Override + public void pushRequestToQueue(TimeseriesDeleteRequest request, List result, FutureCallback callback) { + var tenantId = request.getTenantId(); + var entityId = request.getEntityId(); + + checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matchesKeys(result), cf -> cf.linkMatchesTsKeys(entityId, result), + () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); + } + + private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, + Predicate mainEntityFilter, Predicate linkedEntityFilter, + Supplier msg, FutureCallback callback) { + boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); + if (send) { + clusterService.pushMsgToCalculatedFields(tenantId, entityId, msg.get(), wrap(callback)); + } else { + if (callback != null) { + callback.onSuccess(null); + } + } + } + + private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter) { + boolean send = false; + if (supportedReferencedEntities.contains(entityId.getEntityType())) { + send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(entityId).stream().anyMatch(filter); + if (!send) { + send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(getProfileId(tenantId, entityId)).stream().anyMatch(filter); + } + if (!send) { + send = calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId).stream() + .map(CalculatedFieldLink::getCalculatedFieldId) + .map(calculatedFieldCache::getCalculatedFieldCtx) + .anyMatch(linkedEntityFilter); + } + } + return send; + } + + private EntityId getProfileId(TenantId tenantId, EntityId entityId) { + return switch (entityId.getEntityType()) { + case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); + default -> null; + }; + } + + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { + ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); + + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); + + List entries = request.getEntries(); + List versions = result != null ? result.getVersions() : Collections.emptyList(); + + for (int i = 0; i < entries.size(); i++) { + TsKvProto.Builder tsProtoBuilder = toTsKvProto(entries.get(i)).toBuilder(); + if (result != null) { + tsProtoBuilder.setVersion(versions.get(i)); + } + telemetryMsg.addTsData(tsProtoBuilder.build()); + } + + msg.setTelemetryMsg(telemetryMsg.build()); + return msg.build(); + } + + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List versions) { + ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); + + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); + telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); + List entries = request.getEntries(); + for (int i = 0; i < entries.size(); i++) { + AttributeValueProto.Builder attrProtoBuilder = ProtoUtils.toProto(entries.get(i)).toBuilder(); + if (versions != null) { + attrProtoBuilder.setVersion(versions.get(i)); + } + telemetryMsg.addAttrData(attrProtoBuilder.build()); + } + msg.setTelemetryMsg(telemetryMsg.build()); + + return msg.build(); + } + + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesDeleteRequest request, List removedKeys) { + CalculatedFieldTelemetryMsgProto telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()) + .setScope(AttributeScopeProto.valueOf(request.getScope().name())) + .addAllRemovedAttrKeys(removedKeys).build(); + return ToCalculatedFieldMsg.newBuilder() + .setTelemetryMsg(telemetryMsg) + .build(); + } + + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesDeleteRequest request, List removedKeys) { + CalculatedFieldTelemetryMsgProto telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()) + .addAllRemovedTsKeys(removedKeys).build(); + return ToCalculatedFieldMsg.newBuilder() + .setTelemetryMsg(telemetryMsg) + .build(); + } + + private CalculatedFieldTelemetryMsgProto.Builder buildTelemetryMsgProto(TenantId tenantId, EntityId entityId, List calculatedFieldIds, UUID tbMsgId, TbMsgType tbMsgType) { + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = CalculatedFieldTelemetryMsgProto.newBuilder(); + + telemetryMsg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + telemetryMsg.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + + telemetryMsg.setEntityType(entityId.getEntityType().name()); + telemetryMsg.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + + if (calculatedFieldIds != null) { + for (CalculatedFieldId cfId : calculatedFieldIds) { + telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); + } + } + + if (tbMsgId != null) { + telemetryMsg.setTbMsgIdMSB(tbMsgId.getMostSignificantBits()); + telemetryMsg.setTbMsgIdLSB(tbMsgId.getLeastSignificantBits()); + } + + if (tbMsgType != null) { + telemetryMsg.setTbMsgType(tbMsgType.name()); + } + + return telemetryMsg; + } + + private static TbQueueCallback wrap(FutureCallback callback) { + if (callback != null) { + return new FutureCallbackWrapper(callback); + } else { + return DUMMY_TB_QUEUE_CALLBACK; + } + } + + private static class FutureCallbackWrapper implements TbQueueCallback { + private final FutureCallback callback; + + public FutureCallbackWrapper(FutureCallback callback) { + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + callback.onSuccess(null); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java new file mode 100644 index 0000000000..bb5ef91974 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.cache; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; + +import java.util.Collection; + +public interface CalculatedFieldEntityProfileCache extends ApplicationListener { + + void add(TenantId tenantId, EntityId profileId, EntityId entityId); + + void update(TenantId tenantId, EntityId oldProfileId, EntityId newProfileId, EntityId entityId); + + void evict(TenantId tenantId, EntityId entityId); + + Collection getMyEntityIdsByProfileId(TenantId tenantId, EntityId profileId); + + int getEntityIdPartition(TenantId tenantId, EntityId entityId); +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java new file mode 100644 index 0000000000..4cf62c01b3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java @@ -0,0 +1,93 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.cache; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.TbApplicationEventListener; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; + +@TbRuleEngineComponent +@Service +@Slf4j +@RequiredArgsConstructor +//TODO ashvayka: remove and use TenantEntityProfileCache in each CalculatedFieldManagerMessageProcessor; +public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEventListener implements CalculatedFieldEntityProfileCache { + + private static final Integer UNKNOWN = 0; + private final ConcurrentMap tenantCache = new ConcurrentHashMap<>(); + private final PartitionService partitionService; + private volatile List myPartitions = Collections.emptyList(); + + @Override + protected void onTbApplicationEvent(PartitionChangeEvent event) { + myPartitions = event.getCfPartitions().stream() + .filter(TopicPartitionInfo::isMyPartition) + .map(tpi -> tpi.getPartition().orElse(UNKNOWN)).collect(Collectors.toList()); + //Naive approach that need to be improved. + tenantCache.values().forEach(cache -> cache.setMyPartitions(myPartitions)); + } + + @Override + public void add(TenantId tenantId, EntityId profileId, EntityId entityId) { + var tpi = partitionService.resolve(QueueKey.CF, entityId); + var partition = tpi.getPartition().orElse(UNKNOWN); + tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()) + .add(profileId, entityId, partition, tpi.isMyPartition()); + } + + @Override + public void update(TenantId tenantId, EntityId oldProfileId, EntityId newProfileId, EntityId entityId) { + var tpi = partitionService.resolve(QueueKey.CF, entityId); + var partition = tpi.getPartition().orElse(UNKNOWN); + var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); + //TODO: make this method atomic; + cache.remove(oldProfileId, entityId); + cache.add(newProfileId, entityId, partition, tpi.isMyPartition()); + } + + @Override + public void evict(TenantId tenantId, EntityId entityId) { + var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); + cache.removeEntityId(entityId); + } + + @Override + public Collection getMyEntityIdsByProfileId(TenantId tenantId, EntityId profileId) { + return tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()).getMyEntityIdsByProfileId(profileId); + } + + @Override + public int getEntityIdPartition(TenantId tenantId, EntityId entityId) { + var tpi = partitionService.resolve(QueueKey.CF, entityId); + return tpi.getPartition().orElse(UNKNOWN); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java new file mode 100644 index 0000000000..1a17b9b8be --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java @@ -0,0 +1,122 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.cache; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class TenantEntityProfileCache { + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private final Map>> allEntities = new HashMap<>(); + private final Map> myEntities = new HashMap<>(); + + public void setMyPartitions(List myPartitions) { + lock.writeLock().lock(); + try { + myEntities.clear(); + myPartitions.forEach(partitionId -> { + var map = allEntities.get(partitionId); + if (map != null) { + map.forEach((profileId, entityIds) -> myEntities.computeIfAbsent(profileId, k -> new HashSet<>()).addAll(entityIds)); + } + }); + } finally { + lock.writeLock().unlock(); + } + } + + public void removeProfileId(EntityId profileId) { + lock.writeLock().lock(); + try { + // Remove from allEntities + allEntities.values().forEach(map -> map.remove(profileId)); + // Remove from myEntities + myEntities.remove(profileId); + } finally { + lock.writeLock().unlock(); + } + } + + public void removeEntityId(EntityId entityId) { + lock.writeLock().lock(); + try { + // Remove from allEntities + allEntities.values().forEach(map -> map.values().forEach(set -> set.remove(entityId))); + // Remove from myEntities + myEntities.values().forEach(set -> set.remove(entityId)); + } finally { + lock.writeLock().unlock(); + } + } + + public void remove(EntityId profileId, EntityId entityId) { + lock.writeLock().lock(); + try { + // Remove from allEntities + allEntities.values().forEach(map -> removeSafely(map, profileId, entityId)); + // Remove from myEntities + removeSafely(myEntities, profileId, entityId); + } finally { + lock.writeLock().unlock(); + } + } + + public void add(EntityId profileId, EntityId entityId, Integer partition, boolean mine) { + lock.writeLock().lock(); + try { + if(EntityType.DEVICE.equals(profileId.getEntityType())){ + throw new RuntimeException("WTF?"); + } + if (mine) { + myEntities.computeIfAbsent(profileId, k -> new HashSet<>()).add(entityId); + } + allEntities.computeIfAbsent(partition, k -> new HashMap<>()).computeIfAbsent(profileId, p -> new HashSet<>()).add(entityId); + } finally { + lock.writeLock().unlock(); + } + } + + public Collection getMyEntityIdsByProfileId(EntityId profileId) { + lock.readLock().lock(); + try { + var entities = myEntities.getOrDefault(profileId, Collections.emptySet()); + List result = new ArrayList<>(entities.size()); + result.addAll(entities); + return result; + } finally { + lock.readLock().unlock(); + } + } + + private void removeSafely(Map> map, EntityId profileId, EntityId entityId) { + var set = map.get(profileId); + if (set != null) { + set.remove(entityId); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java new file mode 100644 index 0000000000..6694252652 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx; + +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; + +@Data +@NoArgsConstructor +public class CalculatedFieldEntityCtx { + + private CalculatedFieldEntityCtxId id; + private CalculatedFieldState state; + + public CalculatedFieldEntityCtx(CalculatedFieldEntityCtxId id, CalculatedFieldState state) { + this.id = id; + this.state = state; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java new file mode 100644 index 0000000000..329028eda4 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx; + +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +public record CalculatedFieldEntityCtxId(TenantId tenantId, CalculatedFieldId cfId, EntityId entityId) { + + public String toKey() { + return cfId + "_" + entityId; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java new file mode 100644 index 0000000000..83e10b8194 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = SingleValueArgumentEntry.class, name = "SINGLE_VALUE"), + @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING") +}) +public interface ArgumentEntry { + + @JsonIgnore + ArgumentEntryType getType(); + + Object getValue(); + + boolean updateEntry(ArgumentEntry entry); + + boolean isEmpty(); + + TbelCfArg toTbelCfArg(); + + boolean isForceResetPrevious(); + + void setForceResetPrevious(boolean forceResetPrevious); + + static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { + return new SingleValueArgumentEntry(kvEntry); + } + + static ArgumentEntry createTsRollingArgument(List kvEntries, int limit, long timeWindow) { + return new TsRollingArgumentEntry(kvEntries, limit, timeWindow); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java new file mode 100644 index 0000000000..68f973c7c1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +public enum ArgumentEntryType { + SINGLE_VALUE, TS_ROLLING +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java new file mode 100644 index 0000000000..80b003b3cc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -0,0 +1,103 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.utils.CalculatedFieldUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.thingsboard.server.utils.CalculatedFieldUtils.toSingleValueArgumentProto; + +@Data +@AllArgsConstructor +public abstract class BaseCalculatedFieldState implements CalculatedFieldState { + + protected List requiredArguments; + protected Map arguments; + protected boolean sizeExceedsLimit; + + public BaseCalculatedFieldState(List requiredArguments) { + this.requiredArguments = requiredArguments; + this.arguments = new HashMap<>(); + } + + public BaseCalculatedFieldState() { + this(new ArrayList<>(), new HashMap<>(), false); + } + + @Override + public boolean updateState(CalculatedFieldCtx ctx, Map argumentValues) { + if (arguments == null) { + arguments = new HashMap<>(); + } + + boolean stateUpdated = false; + + for (Map.Entry entry : argumentValues.entrySet()) { + String key = entry.getKey(); + ArgumentEntry newEntry = entry.getValue(); + + checkArgumentSize(key, newEntry, ctx); + + ArgumentEntry existingEntry = arguments.get(key); + + if (existingEntry == null || newEntry.isForceResetPrevious()) { + validateNewEntry(newEntry); + arguments.put(key, newEntry); + stateUpdated = true; + } else { + stateUpdated = existingEntry.updateEntry(newEntry); + } + } + + return stateUpdated; + } + + @Override + public boolean isReady() { + return arguments.keySet().containsAll(requiredArguments) && + arguments.values().stream().noneMatch(ArgumentEntry::isEmpty); + } + + @Override + public void checkStateSize(CalculatedFieldEntityCtxId ctxId, long maxStateSize) { + if (!sizeExceedsLimit && maxStateSize > 0 && CalculatedFieldUtils.toProto(ctxId, this).getSerializedSize() > maxStateSize) { + arguments.clear(); + sizeExceedsLimit = true; + } + } + + @Override + public void checkArgumentSize(String name, ArgumentEntry entry, CalculatedFieldCtx ctx) { + if (entry instanceof TsRollingArgumentEntry) { + return; + } + if (entry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + if (ctx.getMaxSingleValueArgumentSize() > 0 && toSingleValueArgumentProto(name, singleValueArgumentEntry).getSerializedSize() > ctx.getMaxSingleValueArgumentSize()) { + throw new IllegalArgumentException("Single value size exceeds the maximum allowed limit. The argument will not be used for calculation."); + } + } + } + + protected abstract void validateNewEntry(ArgumentEntry newEntry); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java new file mode 100644 index 0000000000..e5a3d0e05e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -0,0 +1,282 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.Data; +import net.objecthunter.exp4j.Expression; +import net.objecthunter.exp4j.ExpressionBuilder; +import org.mvel2.MVEL; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Data +public class CalculatedFieldCtx { + + private CalculatedField calculatedField; + + private CalculatedFieldId cfId; + private TenantId tenantId; + private EntityId entityId; + private CalculatedFieldType cfType; + private final Map arguments; + private final Map mainEntityArguments; + private final Map> linkedEntityArguments; + + private final Map, String> referencedEntityKeys; + private final List argNames; + private Output output; + private String expression; + private TbelInvokeService tbelInvokeService; + private CalculatedFieldScriptEngine calculatedFieldScriptEngine; + private ThreadLocal customExpression; + + private boolean initialized; + + private long maxDataPointsPerRollingArg; + private long maxStateSize; + private long maxSingleValueArgumentSize; + + public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService, ApiLimitService apiLimitService) { + this.calculatedField = calculatedField; + + this.cfId = calculatedField.getId(); + this.tenantId = calculatedField.getTenantId(); + this.entityId = calculatedField.getEntityId(); + this.cfType = calculatedField.getType(); + CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); + this.arguments = configuration.getArguments(); + this.mainEntityArguments = new HashMap<>(); + this.linkedEntityArguments = new HashMap<>(); + for (Map.Entry entry : arguments.entrySet()) { + var refId = entry.getValue().getRefEntityId(); + var refKey = entry.getValue().getRefEntityKey(); + if (refId == null) { + mainEntityArguments.put(refKey, entry.getKey()); + } else { + linkedEntityArguments.computeIfAbsent(refId, key -> new HashMap<>()).put(refKey, entry.getKey()); + } + } + this.referencedEntityKeys = arguments.entrySet().stream() + .collect(Collectors.toMap( + entry -> new TbPair<>(entry.getValue().getRefEntityId() == null ? entityId : entry.getValue().getRefEntityId(), entry.getValue().getRefEntityKey()), + Map.Entry::getKey + )); + this.argNames = new ArrayList<>(arguments.keySet()); + this.output = configuration.getOutput(); + this.expression = configuration.getExpression(); + this.tbelInvokeService = tbelInvokeService; + + this.maxDataPointsPerRollingArg = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); + this.maxStateSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes) * 1024; + this.maxSingleValueArgumentSize = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxSingleValueArgumentSizeInKBytes) * 1024; + } + + public void init() { + if (CalculatedFieldType.SCRIPT.equals(cfType)) { + try { + this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); + initialized = true; + } catch (Exception e) { + throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax.", e); + } + } else { + if (isValidExpression(expression)) { + this.customExpression = ThreadLocal.withInitial(() -> + new ExpressionBuilder(expression) + .implicitMultiplication(true) + .variables(this.arguments.keySet()) + .build() + ); + initialized = true; + } else { + throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax."); + } + } + } + + private CalculatedFieldScriptEngine initEngine(TenantId tenantId, String expression, TbelInvokeService tbelInvokeService) { + if (tbelInvokeService == null) { + throw new IllegalArgumentException("TBEL script engine is disabled!"); + } + + List ctxAndArgNames = new ArrayList<>(argNames.size() + 1); + ctxAndArgNames.add("ctx"); + ctxAndArgNames.addAll(argNames); + return new CalculatedFieldTbelScriptEngine( + tenantId, + tbelInvokeService, + expression, + ctxAndArgNames.toArray(String[]::new) + ); + } + + private boolean isValidExpression(String expression) { + try { + MVEL.compileExpression(expression); + return true; + } catch (Exception e) { + return false; + } + } + + public boolean matches(List values, AttributeScope scope) { + return matchesAttributes(mainEntityArguments, values, scope); + } + + public boolean linkMatches(EntityId entityId, List values, AttributeScope scope) { + var map = linkedEntityArguments.get(entityId); + return map != null && matchesAttributes(map, values, scope); + } + + public boolean matches(List values) { + return matchesTimeSeries(mainEntityArguments, values); + } + + public boolean linkMatches(EntityId entityId, List values) { + var map = linkedEntityArguments.get(entityId); + return map != null && matchesTimeSeries(map, values); + } + + private boolean matchesAttributes(Map argMap, List values, AttributeScope scope) { + for (AttributeKvEntry attrKv : values) { + ReferencedEntityKey attrKey = new ReferencedEntityKey(attrKv.getKey(), ArgumentType.ATTRIBUTE, scope); + if (argMap.containsKey(attrKey)) { + return true; + } + } + return false; + } + + private boolean matchesTimeSeries(Map argMap, List values) { + for (TsKvEntry tsKv : values) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_LATEST, null); + if (argMap.containsKey(latestKey)) { + return true; + } + ReferencedEntityKey rollingKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_ROLLING, null); + if (argMap.containsKey(rollingKey)) { + return true; + } + } + return false; + } + + public boolean matchesKeys(List keys, AttributeScope scope) { + return matchesAttributesKeys(mainEntityArguments, keys, scope); + } + + public boolean matchesKeys(List keys) { + return matchesTimeSeriesKeys(mainEntityArguments, keys); + } + + private boolean matchesAttributesKeys(Map argMap, List keys, AttributeScope scope) { + for (String key : keys) { + ReferencedEntityKey attrKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); + if (argMap.containsKey(attrKey)) { + return true; + } + } + return false; + } + + private boolean matchesTimeSeriesKeys(Map argMap, List keys) { + for (String key : keys) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); + if (argMap.containsKey(latestKey)) { + return true; + } + ReferencedEntityKey rollingKey = new ReferencedEntityKey(key, ArgumentType.TS_ROLLING, null); + if (argMap.containsKey(rollingKey)) { + return true; + } + } + return false; + } + + public boolean linkMatchesAttrKeys(EntityId entityId, List keys, AttributeScope scope) { + var map = linkedEntityArguments.get(entityId); + return map != null && matchesAttributesKeys(map, keys, scope); + } + + public boolean linkMatchesTsKeys(EntityId entityId, List keys) { + var map = linkedEntityArguments.get(entityId); + return map != null && matchesTimeSeriesKeys(map, keys); + } + + public boolean linkMatches(EntityId entityId, CalculatedFieldTelemetryMsgProto proto) { + if (!proto.getTsDataList().isEmpty()) { + List updatedTelemetry = proto.getTsDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return linkMatches(entityId, updatedTelemetry); + } else if (!proto.getAttrDataList().isEmpty()) { + AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); + List updatedTelemetry = proto.getAttrDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return linkMatches(entityId, updatedTelemetry, scope); + } else if (!proto.getRemovedTsKeysList().isEmpty()) { + return linkMatchesTsKeys(entityId, proto.getRemovedTsKeysList()); + } else { + return linkMatchesAttrKeys(entityId, proto.getRemovedAttrKeysList(), AttributeScope.valueOf(proto.getScope().name())); + } + } + + public CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId() { + return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); + } + + public boolean hasOtherSignificantChanges(CalculatedFieldCtx other) { + boolean expressionChanged = !expression.equals(other.expression); + boolean outputChanged = !output.equals(other.output); + return expressionChanged || outputChanged; + } + + public boolean hasStateChanges(CalculatedFieldCtx other) { + boolean typeChanged = !cfType.equals(other.cfType); + boolean argumentsChanged = !arguments.equals(other.arguments); + return typeChanged || argumentsChanged; + } + + public String getSizeExceedsLimitMessage() { + return "Failed to init CF state. State size exceeds limit of " + (maxStateSize / 1024) + "Kb!"; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java new file mode 100644 index 0000000000..caad1e4cfe --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.util.concurrent.ListenableFuture; + +public interface CalculatedFieldScriptEngine { + + ListenableFuture executeScriptAsync(Object[] args); + + ListenableFuture executeJsonAsync(Object[] args); + + void destroy(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java new file mode 100644 index 0000000000..fc4ba513d2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.service.cf.CalculatedFieldResult; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; + +import java.util.List; +import java.util.Map; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE"), + @JsonSubTypes.Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT"), +}) +public interface CalculatedFieldState { + + @JsonIgnore + CalculatedFieldType getType(); + + Map getArguments(); + + void setRequiredArguments(List requiredArguments); + + boolean updateState(CalculatedFieldCtx ctx, Map argumentValues); + + ListenableFuture performCalculation(CalculatedFieldCtx ctx); + + @JsonIgnore + boolean isReady(); + + boolean isSizeExceedsLimit(); + + @JsonIgnore + default boolean isSizeOk() { + return !isSizeExceedsLimit(); + } + + void checkStateSize(CalculatedFieldEntityCtxId ctxId, long maxStateSize); + + void checkArgumentSize(String name, ArgumentEntry entry, CalculatedFieldCtx ctx); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java new file mode 100644 index 0000000000..2ca34b7b17 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -0,0 +1,82 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.ScriptType; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.id.TenantId; + +import javax.script.ScriptException; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +@Slf4j +public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEngine { + + private final TbelInvokeService tbelInvokeService; + + private final UUID scriptId; + private final TenantId tenantId; + + public CalculatedFieldTbelScriptEngine(TenantId tenantId, TbelInvokeService tbelInvokeService, String script, String... argNames) { + this.tenantId = tenantId; + this.tbelInvokeService = tbelInvokeService; + try { + this.scriptId = this.tbelInvokeService.eval(tenantId, ScriptType.CALCULATED_FIELD_SCRIPT, script, argNames).get(); + } catch (Exception e) { + Throwable t = e; + if (e instanceof ExecutionException) { + t = e.getCause(); + } + throw new IllegalArgumentException("Can't compile script: " + t.getMessage(), t); + } + } + + @Override + public ListenableFuture executeScriptAsync(Object[] args) { + log.trace("Executing script async, args {}", args); + return Futures.transformAsync(tbelInvokeService.invokeScript(tenantId, null, this.scriptId, args), + o -> { + try { + return Futures.immediateFuture(o); + } catch (Exception e) { + if (e.getCause() instanceof ScriptException) { + return Futures.immediateFailedFuture(e.getCause()); + } else if (e.getCause() instanceof RuntimeException) { + return Futures.immediateFailedFuture(new ScriptException(e.getCause().getMessage())); + } else { + return Futures.immediateFailedFuture(new ScriptException(e)); + } + } + }, MoreExecutors.directExecutor()); + } + + @Override + public ListenableFuture executeJsonAsync(Object[] args) { + return Futures.transform(executeScriptAsync(args), JacksonUtil::valueToTree, MoreExecutors.directExecutor()); + } + + @Override + public void destroy() { + tbelInvokeService.release(this.scriptId); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java new file mode 100644 index 0000000000..e81fa4d1dc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -0,0 +1,157 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgHeaders; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; +import org.thingsboard.server.queue.common.consumer.QueueStateService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; +import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; + +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.*; + +@Service +@RequiredArgsConstructor +@Slf4j +@ConditionalOnExpression("('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine') && '${queue.type:null}'=='kafka'") +public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldStateService { + + private final TbRuleEngineQueueFactory queueFactory; + private final PartitionService partitionService; + + @Value("${queue.calculated_fields.poll_interval:25}") + private long pollInterval; + + private PartitionedQueueConsumerManager> stateConsumer; + private TbKafkaProducerTemplate> stateProducer; + private QueueStateService, TbProtoQueueMsg> queueStateService; + + private final AtomicInteger counter = new AtomicInteger(); + + @Override + public void init(PartitionedQueueConsumerManager> eventConsumer) { + super.init(eventConsumer); + this.stateConsumer = PartitionedQueueConsumerManager.>create() + .queueKey(QueueKey.CF_STATES) + .topic(partitionService.getTopic(QueueKey.CF_STATES)) + .pollInterval(pollInterval) + .msgPackProcessor((msgs, consumer, config) -> { + for (TbProtoQueueMsg msg : msgs) { + try { + if (msg.getValue() != null) { + processRestoredState(msg.getValue()); + } else { + processRestoredState(getStateId(msg.getHeaders()), null); + } + } catch (Throwable t) { + log.error("Failed to process state message: {}", msg, t); + } + + int processedMsgCount = counter.incrementAndGet(); + if (processedMsgCount % 10000 == 0) { + log.info("Processed {} calculated field state msgs", processedMsgCount); + } + } + }) + .consumerCreator((config, partitionId) -> queueFactory.createCalculatedFieldStateConsumer()) + .consumerExecutor(eventConsumer.getConsumerExecutor()) + .scheduler(eventConsumer.getScheduler()) + .taskExecutor(eventConsumer.getTaskExecutor()) + .build(); + this.stateProducer = (TbKafkaProducerTemplate>) queueFactory.createCalculatedFieldStateProducer(); + this.queueStateService = new QueueStateService<>(); + this.queueStateService.init(stateConsumer, super.eventConsumer); + } + + @Override + protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF_STATES, stateId.entityId()); + TbProtoQueueMsg msg = new TbProtoQueueMsg<>(stateId.entityId().getId(), stateMsgProto); + if (stateMsgProto == null) { + putStateId(msg.getHeaders(), stateId); + } + stateProducer.send(tpi, stateId.toKey(), msg, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (callback != null) { + callback.onSuccess(); + } + } + + @Override + public void onFailure(Throwable t) { + if (callback != null) { + callback.onFailure(t); + } + } + }); + } + + @Override + protected void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback) { + doPersist(stateId, null, callback); + } + + @Override + public void restore(Set partitions) { + queueStateService.update(partitions); + } + + private void putStateId(TbQueueMsgHeaders headers, CalculatedFieldEntityCtxId stateId) { + headers.put("tenantId", uuidToBytes(stateId.tenantId().getId())); + headers.put("cfId", uuidToBytes(stateId.cfId().getId())); + headers.put("entityId", uuidToBytes(stateId.entityId().getId())); + headers.put("entityType", stringToBytes(stateId.entityId().getEntityType().name())); + } + + private CalculatedFieldEntityCtxId getStateId(TbQueueMsgHeaders headers) { + TenantId tenantId = TenantId.fromUUID(bytesToUuid(headers.get("tenantId"))); + CalculatedFieldId cfId = new CalculatedFieldId(bytesToUuid(headers.get("cfId"))); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(bytesToString(headers.get("entityType")), bytesToUuid(headers.get("entityId"))); + return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); + } + + @Override + public void stop() { + stateConsumer.stop(); + stateConsumer.awaitStop(); + stateProducer.stop(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java new file mode 100644 index 0000000000..a508eecada --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java @@ -0,0 +1,73 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.google.protobuf.InvalidProtocolBufferException; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService; +import org.thingsboard.server.service.cf.CfRocksDb; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; + +import java.util.Set; + +@Service +@RequiredArgsConstructor +@Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='in-memory'") +public class RocksDBCalculatedFieldStateService extends AbstractCalculatedFieldStateService { + + private final CfRocksDb cfRocksDb; + + private boolean initialized; + + @Override + protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback) { + cfRocksDb.put(stateId.toKey(), stateMsgProto.toByteArray()); + callback.onSuccess(); + } + + @Override + protected void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback) { + cfRocksDb.delete(stateId.toKey()); + callback.onSuccess(); + } + + @Override + public void restore(Set partitions) { + if (!this.initialized) { + cfRocksDb.forEach((key, value) -> { + try { + processRestoredState(CalculatedFieldStateProto.parseFrom(value)); + } catch (InvalidProtocolBufferException e) { + log.error("[{}] Failed to process restored state", key, e); + } + }); + this.initialized = true; + } + eventConsumer.update(partitions); + } + + @Override + public void stop() { + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java new file mode 100644 index 0000000000..bf00f1b0b1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -0,0 +1,83 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.script.api.tbel.TbelCfCtx; +import org.thingsboard.script.api.tbel.TbelCfSingleValueArg; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.service.cf.CalculatedFieldResult; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +@Data +@Slf4j +@NoArgsConstructor +public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { + + public ScriptCalculatedFieldState(List requiredArguments) { + super(requiredArguments); + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SCRIPT; + } + + @Override + protected void validateNewEntry(ArgumentEntry newEntry) { + } + + @Override + public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { + Map arguments = new LinkedHashMap<>(); + List args = new ArrayList<>(ctx.getArgNames().size() + 1); + args.add(new Object()); // first element is a ctx, but we will set it later; + for (String argName : ctx.getArgNames()) { + var arg = toTbelArgument(argName); + arguments.put(argName, arg); + if (arg instanceof TbelCfSingleValueArg svArg) { + args.add(svArg.getValue()); + } else { + args.add(arg); + } + } + args.set(0, new TbelCfCtx(arguments)); + ListenableFuture resultFuture = ctx.getCalculatedFieldScriptEngine().executeJsonAsync(args.toArray()); + Output output = ctx.getOutput(); + return Futures.transform(resultFuture, + result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), + MoreExecutors.directExecutor() + ); + } + + private TbelCfArg toTbelArgument(String key) { + return arguments.get(key).toTbelCfArg(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java new file mode 100644 index 0000000000..480b334ac3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -0,0 +1,83 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.tbel.TbUtils; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.service.cf.CalculatedFieldResult; + +import java.util.List; +import java.util.Map; + +@Data +@NoArgsConstructor +public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { + + public SimpleCalculatedFieldState(List requiredArguments) { + super(requiredArguments); + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SIMPLE; + } + + @Override + protected void validateNewEntry(ArgumentEntry newEntry) { + if (newEntry instanceof TsRollingArgumentEntry) { + throw new IllegalArgumentException("Rolling argument entry is not supported for simple calculated fields."); + } + } + + @Override + public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { + var expr = ctx.getCustomExpression().get(); + + for (Map.Entry entry : this.arguments.entrySet()) { + try { + BasicKvEntry kvEntry = ((SingleValueArgumentEntry) entry.getValue()).getKvEntryValue(); + expr.setVariable(entry.getKey(), Double.parseDouble(kvEntry.getValueAsString())); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Argument '" + entry.getKey() + "' is not a number."); + } + } + + double expressionResult = expr.evaluate(); + + Output output = ctx.getOutput(); + Object result; + Integer decimals = output.getDecimalsByDefault(); + if (decimals != null) { + if (decimals.equals(0)) { + result = TbUtils.toInt(expressionResult); + } else { + result = TbUtils.toFixed(expressionResult, decimals); + } + } else { + result = expressionResult; + } + + return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), JacksonUtil.valueToTree(Map.of(output.getName(), result)))); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java new file mode 100644 index 0000000000..bfe9eed24f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -0,0 +1,115 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.script.api.tbel.TbelCfSingleValueArg; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class SingleValueArgumentEntry implements ArgumentEntry { + + private long ts; + private BasicKvEntry kvEntryValue; + private Long version; + + private boolean forceResetPrevious; + + public SingleValueArgumentEntry(TsKvProto entry) { + this.ts = entry.getTs(); + if (entry.hasVersion()) { + this.version = entry.getVersion(); + } + this.kvEntryValue = ProtoUtils.fromProto(entry.getKv()); + } + + public SingleValueArgumentEntry(AttributeValueProto entry) { + this.ts = entry.getLastUpdateTs(); + if (entry.hasVersion()) { + this.version = entry.getVersion(); + } + this.kvEntryValue = ProtoUtils.basicKvEntryFromProto(entry); + } + + public SingleValueArgumentEntry(KvEntry entry) { + if (entry instanceof TsKvEntry tsKvEntry) { + this.ts = tsKvEntry.getTs(); + this.version = tsKvEntry.getVersion(); + } else if (entry instanceof AttributeKvEntry attributeKvEntry) { + this.ts = attributeKvEntry.getLastUpdateTs(); + this.version = attributeKvEntry.getVersion(); + } + this.kvEntryValue = ProtoUtils.basicKvEntryFromKvEntry(entry); + } + + public SingleValueArgumentEntry(long ts, BasicKvEntry kvEntryValue, Long version) { + this.ts = ts; + this.kvEntryValue = kvEntryValue; + this.version = version; + } + + @Override + public ArgumentEntryType getType() { + return ArgumentEntryType.SINGLE_VALUE; + } + + @Override + public boolean isEmpty() { + return kvEntryValue == null; + } + + @JsonIgnore + public Object getValue() { + return isEmpty() ? null : kvEntryValue.getValue(); + } + + @Override + public TbelCfArg toTbelCfArg() { + return new TbelCfSingleValueArg(ts, kvEntryValue.getValue()); + } + + @Override + public boolean updateEntry(ArgumentEntry entry) { + if (entry instanceof SingleValueArgumentEntry singleValueEntry) { + if (singleValueEntry.getTs() == this.ts) { + return false; + } + + Long newVersion = singleValueEntry.getVersion(); + if (newVersion == null || this.version == null || newVersion > this.version) { + this.ts = singleValueEntry.getTs(); + this.version = newVersion; + this.kvEntryValue = singleValueEntry.getKvEntryValue(); + return true; + } + } else { + throw new IllegalArgumentException("Unsupported argument entry type for single value argument entry: " + entry.getType()); + } + return false; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java new file mode 100644 index 0000000000..b5a680a072 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -0,0 +1,146 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.script.api.tbel.TbelCfTsDoubleVal; +import org.thingsboard.script.api.tbel.TbelCfTsRollingArg; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Slf4j +public class TsRollingArgumentEntry implements ArgumentEntry { + + private Integer limit; + private Long timeWindow; + private TreeMap tsRecords = new TreeMap<>(); + + private boolean forceResetPrevious; + + public TsRollingArgumentEntry(List kvEntries, int limit, long timeWindow) { + this.limit = limit; + this.timeWindow = timeWindow; + kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry)); + } + + public TsRollingArgumentEntry(TreeMap tsRecords, int limit, long timeWindow) { + this.tsRecords = tsRecords; + this.limit = limit; + this.timeWindow = timeWindow; + } + + public TsRollingArgumentEntry(int limit, long timeWindow) { + this.tsRecords = new TreeMap<>(); + this.limit = limit; + this.timeWindow = timeWindow; + } + + public TsRollingArgumentEntry(Integer limit, Long timeWindow, TreeMap tsRecords) { + this.limit = limit; + this.timeWindow = timeWindow; + this.tsRecords = tsRecords; + } + + @Override + public ArgumentEntryType getType() { + return ArgumentEntryType.TS_ROLLING; + } + + @Override + public boolean isEmpty() { + return tsRecords.isEmpty(); + } + + @JsonIgnore + @Override + public Object getValue() { + return tsRecords; + } + + @Override + public TbelCfArg toTbelCfArg() { + List values = new ArrayList<>(tsRecords.size()); + for (var e : tsRecords.entrySet()) { + values.add(new TbelCfTsDoubleVal(e.getKey(), e.getValue())); + } + return new TbelCfTsRollingArg(timeWindow, values); + } + + @Override + public boolean updateEntry(ArgumentEntry entry) { + if (entry instanceof TsRollingArgumentEntry tsRollingEntry) { + updateTsRollingEntry(tsRollingEntry); + } else if (entry instanceof SingleValueArgumentEntry singleValueEntry) { + updateSingleValueEntry(singleValueEntry); + } else { + throw new IllegalArgumentException("Unsupported argument entry type for rolling argument entry: " + entry.getType()); + } + return true; + } + + private void updateTsRollingEntry(TsRollingArgumentEntry tsRollingEntry) { + for (Map.Entry tsRecordEntry : tsRollingEntry.getTsRecords().entrySet()) { + addTsRecord(tsRecordEntry.getKey(), tsRecordEntry.getValue()); + } + } + + private void updateSingleValueEntry(SingleValueArgumentEntry singleValueEntry) { + addTsRecord(singleValueEntry.getTs(), singleValueEntry.getKvEntryValue()); + } + + private void addTsRecord(Long ts, KvEntry value) { + try { + switch (value.getDataType()) { + case LONG -> value.getLongValue().ifPresent(aLong -> tsRecords.put(ts, aLong.doubleValue())); + case DOUBLE -> value.getDoubleValue().ifPresent(aDouble -> tsRecords.put(ts, aDouble)); + case BOOLEAN -> value.getBooleanValue().ifPresent(aBoolean -> tsRecords.put(ts, aBoolean ? 1.0 : 0.0)); + case STRING -> value.getStrValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); + case JSON -> value.getJsonValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); + } + } catch (Exception e) { + tsRecords.put(ts, Double.NaN); + log.debug("Invalid value '{}' for time series rolling arguments. Only numeric values are supported.", value.getValue()); + } finally { + cleanupExpiredRecords(); + } + } + + private void addTsRecord(Long ts, double value) { + tsRecords.put(ts, value); + cleanupExpiredRecords(); + } + + private void cleanupExpiredRecords() { + if (tsRecords.size() > limit) { + tsRecords.pollFirstEntry(); + } + tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - timeWindow); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java index 6f9845b9a5..c62a551310 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java @@ -82,6 +82,11 @@ public class EdgeEventSourcingListener { @TransactionalEventListener(fallbackExecution = true) public void handleEvent(SaveEntityEvent event) { + if (Boolean.FALSE.equals(event.getBroadcastEvent())) { + log.trace("Ignoring event {}", event); + return; + } + try { if (!isValidSaveEntityEventForEdgeProcessing(event)) { return; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index 9f0e184853..a970097548 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -74,6 +74,8 @@ import org.thingsboard.server.gen.edge.v1.RequestMsgType; import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg; import org.thingsboard.server.gen.edge.v1.ResponseMsg; import org.thingsboard.server.gen.edge.v1.RuleChainMetadataRequestMsg; +import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; +import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; import org.thingsboard.server.gen.edge.v1.SyncCompletedMsg; import org.thingsboard.server.gen.edge.v1.UplinkMsg; import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg; @@ -820,6 +822,16 @@ public abstract class EdgeGrpcSession implements Closeable { result.add(ctx.getAssetProcessor().processAssetMsgFromEdge(edge.getTenantId(), edge, assetUpdateMsg)); } } + if (uplinkMsg.getRuleChainUpdateMsgCount() > 0) { + for (RuleChainUpdateMsg ruleChainUpdateMsg : uplinkMsg.getRuleChainUpdateMsgList()) { + result.add(ctx.getRuleChainProcessor().processRuleChainMsgFromEdge(edge.getTenantId(), edge, ruleChainUpdateMsg)); + } + } + if (uplinkMsg.getRuleChainMetadataUpdateMsgCount() > 0) { + for (RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg : uplinkMsg.getRuleChainMetadataUpdateMsgList()) { + result.add(ctx.getRuleChainProcessor().processRuleChainMetadataMsgFromEdge(edge.getTenantId(), edge, ruleChainMetadataUpdateMsg)); + } + } if (uplinkMsg.getEntityViewUpdateMsgCount() > 0) { for (EntityViewUpdateMsg entityViewUpdateMsg : uplinkMsg.getEntityViewUpdateMsgList()) { result.add(ctx.getEntityViewProcessor().processEntityViewMsgFromEdge(edge.getTenantId(), edge, entityViewUpdateMsg)); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/EdgeEntityProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/EdgeEntityProcessor.java index ddbe4810df..77fa31c028 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/EdgeEntityProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/edge/EdgeEntityProcessor.java @@ -49,8 +49,12 @@ public class EdgeEntityProcessor extends BaseEdgeProcessor { @Override public ListenableFuture processEntityNotification(TenantId tenantId, TransportProtos.EdgeNotificationMsgProto edgeNotificationMsg) { try { + EdgeId originatorEdgeId = safeGetEdgeId(edgeNotificationMsg.getOriginatorEdgeIdMSB(), edgeNotificationMsg.getOriginatorEdgeIdLSB()); EdgeEventActionType actionType = EdgeEventActionType.valueOf(edgeNotificationMsg.getAction()); EdgeId edgeId = new EdgeId(new UUID(edgeNotificationMsg.getEntityIdMSB(), edgeNotificationMsg.getEntityIdLSB())); + if (edgeId.equals(originatorEdgeId)) { + return Futures.immediateFuture(null); + } switch (actionType) { case ASSIGNED_TO_CUSTOMER: { CustomerId customerId = JacksonUtil.fromString(edgeNotificationMsg.getBody(), CustomerId.class); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/BaseRuleChainProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/BaseRuleChainProcessor.java new file mode 100644 index 0000000000..00b9c732aa --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/BaseRuleChainProcessor.java @@ -0,0 +1,82 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.processor.rule; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.util.Pair; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.dao.service.DataValidator; +import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; +import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; +import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; + +import java.util.function.Function; + +@Slf4j +public class BaseRuleChainProcessor extends BaseEdgeProcessor { + + @Autowired + private DataValidator ruleChainValidator; + + protected Pair saveOrUpdateRuleChain(TenantId tenantId, RuleChainId ruleChainId, RuleChainUpdateMsg ruleChainUpdateMsg, RuleChainType ruleChainType) { + boolean created = false; + RuleChain ruleChainFromDb = edgeCtx.getRuleChainService().findRuleChainById(tenantId, ruleChainId); + if (ruleChainFromDb == null) { + created = true; + } + + RuleChain ruleChain = JacksonUtil.fromString(ruleChainUpdateMsg.getEntity(), RuleChain.class, true); + if (ruleChain == null) { + throw new RuntimeException("[{" + tenantId + "}] ruleChainUpdateMsg {" + ruleChainUpdateMsg + "} cannot be converted to rule chain"); + } + boolean isRoot = ruleChain.isRoot(); + if (RuleChainType.CORE.equals(ruleChainType)) { + ruleChain.setRoot(false); + } else { + ruleChain.setRoot(ruleChainFromDb == null ? false : ruleChainFromDb.isRoot()); + } + ruleChain.setType(ruleChainType); + + ruleChainValidator.validate(ruleChain, RuleChain::getTenantId); + if (created) { + ruleChain.setId(ruleChainId); + } + edgeCtx.getRuleChainService().saveRuleChain(ruleChain, true, false); + return Pair.of(created, isRoot); + } + + protected void saveOrUpdateRuleChainMetadata(TenantId tenantId, RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg) { + RuleChainMetaData ruleChainMetadata = JacksonUtil.fromString(ruleChainMetadataUpdateMsg.getEntity(), RuleChainMetaData.class, true); + if (ruleChainMetadata == null) { + throw new RuntimeException("[{" + tenantId + "}] ruleChainMetadataUpdateMsg {" + ruleChainMetadataUpdateMsg + "} cannot be converted to rule chain metadata"); + } + if (!ruleChainMetadata.getNodes().isEmpty()) { + ruleChainMetadata.setVersion(null); + for (RuleNode ruleNode : ruleChainMetadata.getNodes()) { + ruleNode.setRuleChainId(null); + ruleNode.setId(null); + } + edgeCtx.getRuleChainService().saveRuleChainMetaData(tenantId, ruleChainMetadata, Function.identity(), true); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java index 772300bded..06fb4c37a2 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java @@ -15,29 +15,123 @@ */ package org.thingsboard.server.service.edge.rpc.processor.rule; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.springframework.data.util.Pair; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; -import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; + +import java.util.UUID; import static org.thingsboard.server.dao.edge.EdgeServiceImpl.EDGE_IS_ROOT_BODY_KEY; @Slf4j @Component @TbCoreComponent -public class RuleChainEdgeProcessor extends BaseEdgeProcessor { +public class RuleChainEdgeProcessor extends BaseRuleChainProcessor { + + public ListenableFuture processRuleChainMsgFromEdge(TenantId tenantId, Edge edge, RuleChainUpdateMsg ruleChainUpdateMsg) { + log.trace("[{}] executing processRuleChainMsgFromEdge [{}] from edge [{}]", tenantId, ruleChainUpdateMsg, edge.getName()); + RuleChainId ruleChainId = new RuleChainId(new UUID(ruleChainUpdateMsg.getIdMSB(), ruleChainUpdateMsg.getIdLSB())); + try { + edgeSynchronizationManager.getEdgeId().set(edge.getId()); + + switch (ruleChainUpdateMsg.getMsgType()) { + case ENTITY_CREATED_RPC_MESSAGE: + case ENTITY_UPDATED_RPC_MESSAGE: + return saveOrUpdateRuleChain(tenantId, ruleChainId, ruleChainUpdateMsg, edge); + case ENTITY_DELETED_RPC_MESSAGE: + RuleChain ruleChainToDelete = edgeCtx.getRuleChainService().findRuleChainById(tenantId, ruleChainId); + if (ruleChainToDelete != null) { + edgeCtx.getRuleChainService().unassignRuleChainFromEdge(tenantId, ruleChainId, edge.getId(), false); + } + return Futures.immediateFuture(null); + case UNRECOGNIZED: + default: + return handleUnsupportedMsgType(ruleChainUpdateMsg.getMsgType()); + } + } catch (DataValidationException e) { + if (e.getMessage().contains("limit reached")) { + log.warn("[{}] Number of allowed rule chains violated {}", tenantId, ruleChainUpdateMsg, e); + return Futures.immediateFuture(null); + } else { + return Futures.immediateFailedFuture(e); + } + } finally { + edgeSynchronizationManager.getEdgeId().remove(); + } + } + + private ListenableFuture saveOrUpdateRuleChain(TenantId tenantId, RuleChainId ruleChainId, RuleChainUpdateMsg ruleChainUpdateMsg, Edge edge) { + try { + Pair resultPair = super.saveOrUpdateRuleChain(tenantId, ruleChainId, ruleChainUpdateMsg, RuleChainType.EDGE); + Boolean created = resultPair.getFirst(); + if (created) { + createRelationFromEdge(tenantId, edge.getId(), ruleChainId); + pushRuleChainCreatedEventToRuleEngine(tenantId, edge, ruleChainId, ruleChainUpdateMsg.getEntity()); + edgeCtx.getRuleChainService().assignRuleChainToEdge(tenantId, ruleChainId, edge.getId()); + } + Boolean isRoot = resultPair.getSecond(); + if (isRoot) { + edge = edgeCtx.getEdgeService().findEdgeById(tenantId, edge.getId()); + edgeCtx.getEdgeService().setEdgeRootRuleChain(tenantId, edge, ruleChainId); + } + } catch (Exception e) { + log.error("Failed to save or update rule chain", e); + return Futures.immediateFailedFuture(e); + } + return Futures.immediateFuture(null); + } + + private void pushRuleChainCreatedEventToRuleEngine(TenantId tenantId, Edge edge, RuleChainId ruleChainId, String ruleChainAsString) { + try { + TbMsgMetaData msgMetaData = getEdgeActionTbMsgMetaData(edge, null); + pushEntityEventToRuleEngine(tenantId, ruleChainId, null, TbMsgType.ENTITY_CREATED, ruleChainAsString, msgMetaData); + } catch (Exception e) { + log.warn("[{}][{}] Failed to push rule chain action to rule engine: {}", tenantId, ruleChainId, TbMsgType.ENTITY_CREATED.name(), e); + } + } + + public ListenableFuture processRuleChainMetadataMsgFromEdge(TenantId tenantId, Edge edge, RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg) { + log.trace("[{}] executing processRuleChainMetadataMsgFromEdge [{}] from edge [{}]", tenantId, ruleChainMetadataUpdateMsg, edge.getName()); + try { + edgeSynchronizationManager.getEdgeId().set(edge.getId()); + + switch (ruleChainMetadataUpdateMsg.getMsgType()) { + case ENTITY_CREATED_RPC_MESSAGE: + case ENTITY_UPDATED_RPC_MESSAGE: + saveOrUpdateRuleChainMetadata(tenantId, ruleChainMetadataUpdateMsg); + return Futures.immediateFuture(null); + case UNRECOGNIZED: + default: + return handleUnsupportedMsgType(ruleChainMetadataUpdateMsg.getMsgType()); + } + } catch (Exception e) { + String errMsg = String.format("Can't process rule chain metadata update msg %s", ruleChainMetadataUpdateMsg); + log.error(errMsg, e); + return Futures.immediateFailedFuture(new RuntimeException(errMsg, e)); + } finally { + edgeSynchronizationManager.getEdgeId().remove(); + } + } @Override public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent) { diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java index a371582409..6fc6d5bad9 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java @@ -152,8 +152,7 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService { entityData = new HashMap<>(); attributes = JacksonUtil.newObjectNode(); for (AttributeKvEntry attr : ssAttributes) { - if (DefaultDeviceStateService.PERSISTENT_ATTRIBUTES.contains(attr.getKey()) - && !DefaultDeviceStateService.INACTIVITY_TIMEOUT.equals(attr.getKey())) { + if (DefaultDeviceStateService.ACTIVITY_KEYS_WITHOUT_INACTIVITY_TIMEOUT.contains(attr.getKey())) { continue; } if (attr.getDataType() == DataType.BOOLEAN && attr.getBooleanValue().isPresent()) { @@ -200,7 +199,7 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService { } Map> tsData = new HashMap<>(); for (TsKvEntry tsKvEntry : tsKvEntries) { - if (DefaultDeviceStateService.PERSISTENT_ATTRIBUTES.contains(tsKvEntry.getKey())) { + if (DefaultDeviceStateService.ACTIVITY_KEYS_WITH_INACTIVITY_TIMEOUT.contains(tsKvEntry.getKey())) { continue; } tsData.computeIfAbsent(tsKvEntry.getTs(), k -> new HashMap<>()).put(tsKvEntry.getKey(), tsKvEntry.getValue()); diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java new file mode 100644 index 0000000000..51c963ed2f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java @@ -0,0 +1,117 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.edqs.query.EdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsResponse; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.edqs.EdqsApiService; +import org.thingsboard.server.edqs.state.EdqsPartitionService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.provider.EdqsClientQueueFactory; + +import java.util.UUID; + +@Service +@Slf4j +@RequiredArgsConstructor +@ConditionalOnExpression("'${queue.edqs.api.supported:true}' == 'true' && ('${service.type:null}' == 'monolith' || '${service.type:null}' == 'tb-core')") +public class DefaultEdqsApiService implements EdqsApiService { + + private final EdqsPartitionService edqsPartitionService; + private final EdqsClientQueueFactory queueFactory; + private TbQueueRequestTemplate, TbProtoQueueMsg> requestTemplate; + + @Value("${queue.edqs.api.auto_enable:true}") + private boolean autoEnable; + + private Boolean apiEnabled = null; + + @PostConstruct + private void init() { + requestTemplate = queueFactory.createEdqsRequestTemplate(); + requestTemplate.init(); + } + + @Override + public ListenableFuture processRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) { + var requestMsg = ToEdqsMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setTs(System.currentTimeMillis()) + .setRequestMsg(TransportProtos.EdqsRequestMsg.newBuilder() + .setValue(JacksonUtil.toString(request)) + .build()); + if (customerId != null && !customerId.isNullUid()) { + requestMsg.setCustomerIdMSB(customerId.getId().getMostSignificantBits()); + requestMsg.setCustomerIdLSB(customerId.getId().getLeastSignificantBits()); + } + + Integer partition = edqsPartitionService.resolvePartition(tenantId); + ListenableFuture> resultFuture = requestTemplate.send(new TbProtoQueueMsg<>(UUID.randomUUID(), requestMsg.build()), partition); + return Futures.transform(resultFuture, msg -> { + TransportProtos.EdqsResponseMsg responseMsg = msg.getValue().getResponseMsg(); + return JacksonUtil.fromString(responseMsg.getValue(), EdqsResponse.class); + }, MoreExecutors.directExecutor()); + } + + @Override + public boolean isEnabled() { + return Boolean.TRUE.equals(apiEnabled); + } + + @Override + public void setEnabled(boolean enabled) { + if (enabled) { + log.info("Enabling EDQS API"); + } else { + log.info("Disabling EDQS API"); + } + apiEnabled = enabled; + } + + @Override + public boolean isSupported() { + return true; + } + + @Override + public boolean isAutoEnable() { + return autoEnable; + } + + @PreDestroy + private void stop() { + requestTemplate.stop(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java new file mode 100644 index 0000000000..e823dee4e7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java @@ -0,0 +1,298 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import com.google.protobuf.ByteString; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.EdqsSyncRequest; +import org.thingsboard.server.common.data.edqs.Entity; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.msg.edqs.EdqsApiService; +import org.thingsboard.server.common.msg.edqs.EdqsService; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.edqs.processor.EdqsProducer; +import org.thingsboard.server.edqs.state.EdqsPartitionService; +import org.thingsboard.server.edqs.util.EdqsConverter; +import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsCoreServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.discovery.HashPartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.environment.DistributedLock; +import org.thingsboard.server.queue.environment.DistributedLockService; +import org.thingsboard.server.queue.provider.EdqsClientQueueFactory; +import org.thingsboard.server.queue.util.AfterStartUp; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + +@Service +@RequiredArgsConstructor +@Slf4j +@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "true") +public class DefaultEdqsService implements EdqsService { + + private final EdqsClientQueueFactory queueFactory; + private final EdqsConverter edqsConverter; + private final EdqsSyncService edqsSyncService; + private final EdqsApiService edqsApiService; + private final DistributedLockService distributedLockService; + private final AttributesService attributesService; + private final EdqsPartitionService edqsPartitionService; + private final TopicService topicService; + private final TbServiceInfoProvider serviceInfoProvider; + @Autowired @Lazy + private TbClusterService clusterService; + @Autowired @Lazy + private HashPartitionService hashPartitionService; + + private EdqsProducer eventsProducer; + private ExecutorService executor; + private DistributedLock syncLock; + + @PostConstruct + private void init() { + executor = ThingsBoardExecutors.newWorkStealingPool(12, getClass()); + eventsProducer = EdqsProducer.builder() + .queue(EdqsQueue.EVENTS) + .partitionService(edqsPartitionService) + .topicService(topicService) + .producer(queueFactory.createEdqsMsgProducer(EdqsQueue.EVENTS)) + .build(); + syncLock = distributedLockService.getLock("edqs_sync"); + } + + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void onStartUp() { + if (!serviceInfoProvider.isService(ServiceType.TB_CORE)) { + return; + } + executor.submit(() -> { + try { + EdqsSyncState syncState = getSyncState(); + if (edqsSyncService.isSyncNeeded() || syncState == null || syncState.getStatus() != EdqsSyncStatus.FINISHED) { + if (hashPartitionService.isSystemPartitionMine(ServiceType.TB_CORE)) { + processSystemRequest(ToCoreEdqsRequest.builder() + .syncRequest(new EdqsSyncRequest()) + .build()); + } + } else if (edqsApiService.isSupported() && edqsApiService.isAutoEnable()) { + // only if topic/RocksDB is not empty and sync is finished + edqsApiService.setEnabled(true); + } + } catch (Throwable e) { + log.error("Failed to start EDQS service", e); + } + }); + } + + @Override + public void processSystemRequest(ToCoreEdqsRequest request) { + log.info("Processing system request {}", request); + if (request.getSyncRequest() != null) { + saveSyncState(EdqsSyncStatus.REQUESTED); + } + broadcast(request.toInternalMsg()); + } + + @Override + public void processSystemMsg(ToCoreEdqsMsg msg) { + executor.submit(() -> { + log.info("Processing system msg {}", msg); + try { + if (msg.getApiEnabled() != null) { + edqsApiService.setEnabled(msg.getApiEnabled()); + } + + if (msg.getSyncRequest() != null) { + syncLock.lock(); + try { + EdqsSyncState syncState = getSyncState(); + if (syncState != null && syncState.getStatus() == EdqsSyncStatus.FINISHED) { + log.info("EDQS sync is already finished"); + return; + } + + saveSyncState(EdqsSyncStatus.STARTED); + edqsSyncService.sync(); + saveSyncState(EdqsSyncStatus.FINISHED); + + if (edqsApiService.isSupported()) + if (edqsApiService.isAutoEnable()) { + log.info("EDQS sync is finished, auto-enabling API"); + broadcast(ToCoreEdqsMsg.builder() + .apiEnabled(Boolean.TRUE) + .build()); + } else { + log.info("EDQS sync is finished, but leaving API disabled"); + } + } catch (Exception e) { + log.error("Failed to complete sync", e); + saveSyncState(EdqsSyncStatus.FAILED); + } finally { + syncLock.unlock(); + } + } + } catch (Throwable e) { + log.error("Failed to process msg {}", msg, e); + } + }); + } + + @Override + public void onUpdate(TenantId tenantId, EntityId entityId, Object entity) { + EntityType entityType = entityId.getEntityType(); + ObjectType objectType = ObjectType.fromEntityType(entityType); + if (!isEdqsType(tenantId, objectType)) { + log.trace("[{}][{}] Ignoring update event, type {} not supported", tenantId, entityId, entityType); + return; + } + onUpdate(tenantId, objectType, edqsConverter.toEntity(entityType, entity)); + } + + @Override + public void onUpdate(TenantId tenantId, ObjectType objectType, EdqsObject object) { + processEvent(tenantId, objectType, EdqsEventType.UPDATED, object); + } + + @Override + public void onDelete(TenantId tenantId, EntityId entityId) { + EntityType entityType = entityId.getEntityType(); + ObjectType objectType = ObjectType.fromEntityType(entityType); + if (!isEdqsType(tenantId, objectType)) { + log.trace("[{}][{}] Ignoring deletion event, type {} not supported", tenantId, entityId, entityType); + return; + } + onDelete(tenantId, objectType, new Entity(entityType, entityId.getId(), Long.MAX_VALUE)); + } + + @Override + public void onDelete(TenantId tenantId, ObjectType objectType, EdqsObject object) { + processEvent(tenantId, objectType, EdqsEventType.DELETED, object); + } + + protected void processEvent(TenantId tenantId, ObjectType objectType, EdqsEventType eventType, EdqsObject object) { + executor.submit(() -> { + try { + String key = object.key(); + Long version = object.version(); + EdqsEventMsg.Builder eventMsg = EdqsEventMsg.newBuilder() + .setKey(key) + .setObjectType(objectType.name()) + .setData(ByteString.copyFrom(edqsConverter.serialize(objectType, object))) + .setEventType(eventType.name()); + if (version != null) { + eventMsg.setVersion(version); + } + eventsProducer.send(tenantId, objectType, key, ToEdqsMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setTs(System.currentTimeMillis()) + .setEventMsg(eventMsg) + .build()); + } catch (Throwable e) { + log.error("[{}] Failed to push {} event for {} {}", tenantId, eventType, objectType, object, e); + } + }); + } + + private boolean isEdqsType(TenantId tenantId, ObjectType objectType) { + if (objectType == null) { + return false; + } + if (!tenantId.isSysTenantId()) { + return ObjectType.edqsTypes.contains(objectType); + } else { + return ObjectType.edqsSystemTypes.contains(objectType); + } + } + + private void broadcast(ToCoreEdqsMsg msg) { + clusterService.broadcastToCore(ToCoreNotificationMsg.newBuilder() + .setToEdqsCoreServiceMsg(ToEdqsCoreServiceMsg.newBuilder() + .setValue(ByteString.copyFrom(JacksonUtil.writeValueAsBytes(msg)))) + .build()); + } + + @SneakyThrows + private EdqsSyncState getSyncState() { + EdqsSyncState state = attributesService.find(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, AttributeScope.SERVER_SCOPE, "edqsSyncState").get(30, TimeUnit.SECONDS) + .flatMap(KvEntry::getJsonValue) + .map(value -> JacksonUtil.fromString(value, EdqsSyncState.class)) + .orElse(null); + log.info("EDQS sync state: {}", state); + return state; + } + + @SneakyThrows + private void saveSyncState(EdqsSyncStatus status) { + EdqsSyncState state = new EdqsSyncState(status); + log.info("New EDQS sync state: {}", state); + attributesService.save(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, AttributeScope.SERVER_SCOPE, new BaseAttributeKvEntry( + new JsonDataEntry("edqsSyncState", JacksonUtil.toString(state)), + System.currentTimeMillis())).get(30, TimeUnit.SECONDS); + } + + @PreDestroy + private void stop() { + executor.shutdown(); + eventsProducer.stop(); + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + private static class EdqsSyncState { + private EdqsSyncStatus status; + } + + private enum EdqsSyncStatus { + REQUESTED, + STARTED, + FINISHED, + FAILED + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java new file mode 100644 index 0000000000..d77df5ced8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.springframework.transaction.event.TransactionalEventListener; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.msg.edqs.EdqsService; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; +import org.thingsboard.server.dao.eventsourcing.RelationActionEvent; +import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; + +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "true") +public class EdqsListener { + + private final EdqsService edqsService; + + @TransactionalEventListener(fallbackExecution = true) + public void onUpdate(SaveEntityEvent event) { + if (event.getEntityId() == null || event.getEntity() == null) { + return; + } + edqsService.onUpdate(event.getTenantId(), event.getEntityId(), event.getEntity()); + } + + @TransactionalEventListener(fallbackExecution = true) + public void onDelete(DeleteEntityEvent event) { + if (event.getEntityId() == null) { + return; + } + edqsService.onDelete(event.getTenantId(), event.getEntityId()); + } + + @TransactionalEventListener(fallbackExecution = true) + public void handleEvent(RelationActionEvent relationEvent) { + if (relationEvent.getActionType() == ActionType.RELATION_ADD_OR_UPDATE) { + edqsService.onUpdate(relationEvent.getTenantId(), ObjectType.RELATION, relationEvent.getRelation()); + } else if (relationEvent.getActionType() == ActionType.RELATION_DELETED) { + edqsService.onDelete(relationEvent.getTenantId(), ObjectType.RELATION, relationEvent.getRelation()); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java new file mode 100644 index 0000000000..79e0e60983 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java @@ -0,0 +1,284 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.AttributeKv; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.Entity; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.attributes.AttributesDao; +import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; +import org.thingsboard.server.dao.entity.EntityDaoRegistry; +import org.thingsboard.server.dao.model.sql.AttributeKvEntity; +import org.thingsboard.server.dao.model.sql.RelationEntity; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; +import org.thingsboard.server.dao.sql.relation.RelationRepository; +import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; + +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.thingsboard.server.common.data.ObjectType.ATTRIBUTE_KV; +import static org.thingsboard.server.common.data.ObjectType.LATEST_TS_KV; +import static org.thingsboard.server.common.data.ObjectType.RELATION; +import static org.thingsboard.server.common.data.ObjectType.edqsTenantTypes; + +@Slf4j +public abstract class EdqsSyncService { + + @Value("${queue.edqs.sync.entity_batch_size:10000}") + private int entityBatchSize; + @Value("${queue.edqs.sync.ts_batch_size:10000}") + private int tsBatchSize; + @Autowired + private EntityDaoRegistry entityDaoRegistry; + @Autowired + private AttributesDao attributesDao; + @Autowired + private KeyDictionaryDao keyDictionaryDao; + @Autowired + private RelationRepository relationRepository; + @Autowired + private TsKvLatestRepository tsKvLatestRepository; + @Autowired + @Lazy + private DefaultEdqsService edqsService; + + private final ConcurrentHashMap entityInfoMap = new ConcurrentHashMap<>(); + private final ConcurrentHashMap keys = new ConcurrentHashMap<>(); + + private final Map counters = new ConcurrentHashMap<>(); + + public abstract boolean isSyncNeeded(); + + public void sync() { + log.info("Synchronizing data to EDQS"); + long startTs = System.currentTimeMillis(); + counters.clear(); + + syncTenantEntities(); + syncRelations(); + loadKeyDictionary(); + syncAttributes(); + syncLatestTimeseries(); + + counters.clear(); + log.info("Finishing synchronizing data to EDQS in {} ms", (System.currentTimeMillis() - startTs)); + } + + private void process(TenantId tenantId, ObjectType type, EdqsObject object) { + AtomicInteger counter = counters.computeIfAbsent(type, t -> new AtomicInteger()); + if (counter.incrementAndGet() % 10000 == 0) { + log.info("Processed {} {} objects", counter.get(), type); + } + edqsService.processEvent(tenantId, type, EdqsEventType.UPDATED, object); + } + + private void syncTenantEntities() { + for (ObjectType type : edqsTenantTypes) { + log.info("Synchronizing {} entities to EDQS", type); + long ts = System.currentTimeMillis(); + EntityType entityType = type.toEntityType(); + Dao dao = entityDaoRegistry.getDao(entityType); + UUID lastId = UUID.fromString("00000000-0000-0000-0000-000000000000"); + while (true) { + var batch = dao.findNextBatch(lastId, entityBatchSize); + if (batch.isEmpty()) { + break; + } + for (EntityFields entityFields : batch) { + TenantId tenantId = TenantId.fromUUID(entityFields.getTenantId()); + entityInfoMap.put(entityFields.getId(), new EntityIdInfo(entityType, tenantId)); + process(tenantId, type, new Entity(entityType, entityFields)); + } + EntityFields lastRecord = batch.get(batch.size() - 1); + lastId = lastRecord.getId(); + } + log.info("Finished synchronizing {} entities to EDQS in {} ms", type, (System.currentTimeMillis() - ts)); + } + } + + private void syncRelations() { + log.info("Synchronizing relations to EDQS"); + long ts = System.currentTimeMillis(); + UUID lastFromEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); + String lastFromEntityType = ""; + String lastRelationTypeGroup = ""; + String lastRelationType = ""; + UUID lastToEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); + String lastToEntityType = ""; + + while (true) { + List batch = relationRepository.findNextBatch(lastFromEntityId, lastFromEntityType, lastRelationTypeGroup, + lastRelationType, lastToEntityId, lastToEntityType, entityBatchSize); + if (batch.isEmpty()) { + break; + } + processRelationBatch(batch); + + RelationEntity lastRecord = batch.get(batch.size() - 1); + lastFromEntityId = lastRecord.getFromId(); + lastFromEntityType = lastRecord.getFromType(); + lastRelationTypeGroup = lastRecord.getRelationTypeGroup(); + lastRelationType = lastRecord.getRelationType(); + lastToEntityId = lastRecord.getToId(); + lastToEntityType = lastRecord.getToType(); + } + log.info("Finished synchronizing relations to EDQS in {} ms", (System.currentTimeMillis() - ts)); + } + + private void processRelationBatch(List relations) { + for (RelationEntity relation : relations) { + if (RelationTypeGroup.COMMON.name().equals(relation.getRelationTypeGroup())) { + EntityIdInfo entityIdInfo = entityInfoMap.get(relation.getFromId()); + if (entityIdInfo != null) { + process(entityIdInfo.tenantId(), RELATION, relation.toData()); + } else { + log.info("Relation from id not found: {} ", relation); + } + } + } + } + + private void loadKeyDictionary() { + log.info("Loading key dictionary"); + long ts = System.currentTimeMillis(); + var keyDictionaryEntries = new PageDataIterable<>(keyDictionaryDao::findAll, 10000); + for (KeyDictionaryEntry keyDictionaryEntry : keyDictionaryEntries) { + keys.put(keyDictionaryEntry.getKeyId(), keyDictionaryEntry.getKey()); + } + log.info("Finished loading key dictionary in {} ms", (System.currentTimeMillis() - ts)); + } + + private void syncAttributes() { + log.info("Synchronizing attributes to EDQS"); + long ts = System.currentTimeMillis(); + + UUID lastEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); + int lastAttributeType = Integer.MIN_VALUE; + int lastAttributeKey = Integer.MIN_VALUE; + + while (true) { + List batch = attributesDao.findNextBatch(lastEntityId, lastAttributeType, lastAttributeKey, tsBatchSize); + if (batch.isEmpty()) { + break; + } + processAttributeBatch(batch); + + AttributeKvEntity lastRecord = batch.get(batch.size() - 1); + lastEntityId = lastRecord.getId().getEntityId(); + lastAttributeType = lastRecord.getId().getAttributeType(); + lastAttributeKey = lastRecord.getId().getAttributeKey(); + } + log.info("Finished synchronizing attributes to EDQS in {} ms", (System.currentTimeMillis() - ts)); + } + + private void processAttributeBatch(List batch) { + for (AttributeKvEntity attribute : batch) { + attribute.setStrKey(getStrKeyOrFetchFromDb(attribute.getId().getAttributeKey())); + UUID entityId = attribute.getId().getEntityId(); + EntityIdInfo entityIdInfo = entityInfoMap.get(entityId); + if (entityIdInfo == null) { + log.debug("Skipping attribute with entity UUID {} as it is not found in entityInfoMap", entityId); + continue; + } + AttributeKv attributeKv = new AttributeKv( + EntityIdFactory.getByTypeAndUuid(entityIdInfo.entityType(), entityId), + AttributeScope.valueOf(attribute.getId().getAttributeType()), + attribute.toData(), + attribute.getVersion()); + process(entityIdInfo.tenantId(), ATTRIBUTE_KV, attributeKv); + } + } + + private void syncLatestTimeseries() { + log.info("Synchronizing latest timeseries to EDQS"); + long ts = System.currentTimeMillis(); + UUID lastEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); + int lastKey = Integer.MIN_VALUE; + + while (true) { + List batch = tsKvLatestRepository.findNextBatch(lastEntityId, lastKey, tsBatchSize); + if (batch.isEmpty()) { + break; + } + processTsKvLatestBatch(batch); + + TsKvLatestEntity lastRecord = batch.get(batch.size() - 1); + lastEntityId = lastRecord.getEntityId(); + lastKey = lastRecord.getKey(); + } + log.info("Finished synchronizing latest timeseries to EDQS in {} ms", (System.currentTimeMillis() - ts)); + } + + private void processTsKvLatestBatch(List tsKvLatestEntities) { + for (TsKvLatestEntity tsKvLatestEntity : tsKvLatestEntities) { + try { + String strKey = getStrKeyOrFetchFromDb(tsKvLatestEntity.getKey()); + if (strKey == null) { + log.debug("Skipping latest timeseries with key {} as it is not found in key dictionary", tsKvLatestEntity.getKey()); + continue; + } + tsKvLatestEntity.setStrKey(strKey); + UUID entityUuid = tsKvLatestEntity.getEntityId(); + EntityIdInfo entityIdInfo = entityInfoMap.get(entityUuid); + if (entityIdInfo != null) { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityIdInfo.entityType(), entityUuid); + LatestTsKv latestTsKv = new LatestTsKv(entityId, tsKvLatestEntity.toData(), tsKvLatestEntity.getVersion()); + process(entityIdInfo.tenantId(), LATEST_TS_KV, latestTsKv); + } + } catch (Exception e) { + log.error("Failed to sync latest timeseries: {}", tsKvLatestEntity, e); + } + } + } + + private String getStrKeyOrFetchFromDb(int key) { + String strKey = keys.get(key); + if (strKey != null) { + return strKey; + } else { + strKey = keyDictionaryDao.getKey(key); + if (strKey != null) { + keys.put(key, strKey); + } + } + return strKey; + } + + public record EntityIdInfo(EntityType entityType, TenantId tenantId) { + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java new file mode 100644 index 0000000000..4ef552521b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; + +import java.util.Collections; + +@Service +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'kafka'") +public class KafkaEdqsSyncService extends EdqsSyncService { + + private final boolean syncNeeded; + + public KafkaEdqsSyncService(TbKafkaSettings kafkaSettings) { + TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, Collections.emptyMap()); + this.syncNeeded = kafkaAdmin.isTopicEmpty(EdqsQueue.EVENTS.getTopic()); + } + + @Override + public boolean isSyncNeeded() { + return syncNeeded; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java new file mode 100644 index 0000000000..904391f172 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.edqs.util.EdqsRocksDb; + +@Service +@RequiredArgsConstructor +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'in-memory'") +public class LocalEdqsSyncService extends EdqsSyncService { + + private final EdqsRocksDb db; + + @Override + public boolean isSyncNeeded() { + return db.isNew(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index aff013ca2b..e9ee7202a7 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -31,9 +31,15 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; +import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; +import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; @@ -71,6 +77,8 @@ public abstract class AbstractTbEntityService { @Autowired(required = false) @Lazy private EntitiesVersionControlService vcService; + @Autowired + protected EntityService entityService; protected boolean isTestProfile() { return Set.of(this.env.getActiveProfiles()).contains("test"); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 35aad36dff..648e89adc9 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -31,7 +31,9 @@ import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.DeviceId; @@ -50,10 +52,12 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceCredentialsUpdateNotificationMsg; +import org.thingsboard.server.dao.edge.EdgeSynchronizationManager; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.TbQueueCallback; import java.util.Set; @@ -64,6 +68,7 @@ public class EntityStateSourcingListener { private final TenantService tenantService; private final TbClusterService tbClusterService; + private final EdgeSynchronizationManager edgeSynchronizationManager; @PostConstruct public void init() { @@ -72,6 +77,11 @@ public class EntityStateSourcingListener { @TransactionalEventListener(fallbackExecution = true) public void handleEvent(SaveEntityEvent event) { + if (Boolean.FALSE.equals(event.getBroadcastEvent())) { + log.trace("Ignoring event {}", event); + return; + } + TenantId tenantId = event.getTenantId(); EntityId entityId = event.getEntityId(); if (entityId == null) { @@ -83,7 +93,10 @@ public class EntityStateSourcingListener { ComponentLifecycleEvent lifecycleEvent = isCreated ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED; switch (entityType) { - case ASSET, ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE -> { + case ASSET -> { + onAssetUpdate(event.getEntity(), event.getOldEntity()); + } + case ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, lifecycleEvent); } case RULE_CHAIN -> { @@ -118,7 +131,11 @@ public class EntityStateSourcingListener { ApiUsageState apiUsageState = (ApiUsageState) event.getEntity(); tbClusterService.onApiStateChange(apiUsageState, null); } - default -> {} + case CALCULATED_FIELD -> { + onCalculatedFieldUpdate(event.getEntity(), event.getOldEntity()); + } + default -> { + } } } @@ -130,14 +147,18 @@ public class EntityStateSourcingListener { return; } EntityType entityType = entityId.getEntityType(); - if (!tenantId.isSysTenantId() && entityType != EntityType.TENANT && !tenantService.tenantExists(tenantId)) { + if (!tenantId.isSysTenantId() && entityType != EntityType.TENANT && !tenantService.tenantExists(tenantId)) { log.debug("[{}] Ignoring DeleteEntityEvent because tenant does not exist: {}", tenantId, event); return; } log.debug("[{}][{}][{}] Handling entity deletion event: {}", tenantId, entityType, entityId, event); switch (entityType) { - case ASSET, ASSET_PROFILE, EDGE, ENTITY_VIEW, CUSTOMER, NOTIFICATION_RULE -> { + case ASSET -> { + Asset asset = (Asset) event.getEntity(); + tbClusterService.onAssetDeleted(tenantId, asset, null); + } + case ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); } case NOTIFICATION_REQUEST -> { @@ -149,7 +170,8 @@ public class EntityStateSourcingListener { case RULE_CHAIN -> { RuleChain ruleChain = (RuleChain) event.getEntity(); if (RuleChainType.CORE.equals(ruleChain.getType())) { - Set referencingRuleChainIds = JacksonUtil.fromString(event.getBody(), new TypeReference<>() {}); + Set referencingRuleChainIds = JacksonUtil.fromString(event.getBody(), new TypeReference<>() { + }); if (referencingRuleChainIds != null) { referencingRuleChainIds.forEach(referencingRuleChainId -> tbClusterService.broadcastEntityStateChangeEvent(tenantId, referencingRuleChainId, ComponentLifecycleEvent.UPDATED)); @@ -163,11 +185,11 @@ public class EntityStateSourcingListener { } case TENANT_PROFILE -> { TenantProfile tenantProfile = (TenantProfile) event.getEntity(); - tbClusterService.onTenantProfileDelete(tenantProfile, null); + tbClusterService.onTenantProfileDelete(tenantProfile, TbQueueCallback.EMPTY); } case DEVICE -> { Device device = (Device) event.getEntity(); - tbClusterService.onDeviceDeleted(tenantId, device, null); + tbClusterService.onDeviceDeleted(tenantId, device, TbQueueCallback.EMPTY); } case DEVICE_PROFILE -> { DeviceProfile deviceProfile = (DeviceProfile) event.getEntity(); @@ -175,9 +197,14 @@ public class EntityStateSourcingListener { } case TB_RESOURCE -> { TbResourceInfo tbResource = (TbResourceInfo) event.getEntity(); - tbClusterService.onResourceDeleted(tbResource, null); + tbClusterService.onResourceDeleted(tbResource, TbQueueCallback.EMPTY); + } + case CALCULATED_FIELD -> { + CalculatedField calculatedField = (CalculatedField) event.getEntity(); + tbClusterService.onCalculatedFieldDeleted(calculatedField, TbQueueCallback.EMPTY); + } + default -> { } - default -> {} } } @@ -239,14 +266,35 @@ public class EntityStateSourcingListener { tbClusterService.onDeviceUpdated(device, oldDevice); } + private void onAssetUpdate(Object entity, Object oldEntity) { + Asset asset = (Asset) entity; + Asset oldAsset = null; + if (oldEntity instanceof Asset) { + oldAsset = (Asset) oldEntity; + } + tbClusterService.onAssetUpdated(asset, oldAsset); + } + private void onEdgeEvent(TenantId tenantId, EntityId entityId, Object entity, ComponentLifecycleEvent lifecycleEvent) { if (entity instanceof Edge) { + if (entityId.equals(edgeSynchronizationManager.getEdgeId().get())) { + return; + } tbClusterService.onEdgeStateChangeEvent(new ComponentLifecycleMsg(tenantId, entityId, lifecycleEvent)); } else if (entity instanceof EdgeEvent edgeEvent) { tbClusterService.onEdgeEventUpdate(new EdgeEventUpdateMsg(tenantId, edgeEvent.getEdgeId())); } } + private void onCalculatedFieldUpdate(Object entity, Object oldEntity) { + CalculatedField calculatedField = (CalculatedField) entity; + CalculatedField oldCalculatedField = null; + if (oldEntity instanceof CalculatedField) { + oldCalculatedField = (CalculatedField) oldEntity; + } + tbClusterService.onCalculatedFieldUpdated(calculatedField, oldCalculatedField, TbQueueCallback.EMPTY); + } + private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) { String data = JacksonUtil.toString(JacksonUtil.valueToTree(assignedDevice)); if (data != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java new file mode 100644 index 0000000000..4dfaec91cf --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -0,0 +1,106 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.entitiy.AbstractTbEntityService; +import org.thingsboard.server.service.security.model.SecurityUser; + +import java.util.Optional; + +@TbCoreComponent +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { + + private final CalculatedFieldService calculatedFieldService; + + @Override + public CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException { + ActionType actionType = calculatedField.getId() == null ? ActionType.ADDED : ActionType.UPDATED; + TenantId tenantId = calculatedField.getTenantId(); + try { + if (ActionType.UPDATED.equals(actionType)) { + CalculatedField existingCf = calculatedFieldService.findById(tenantId, calculatedField.getId()); + checkForEntityChange(existingCf, calculatedField); + } + checkEntityExistence(tenantId, calculatedField.getEntityId()); + CalculatedField savedCalculatedField = checkNotNull(calculatedFieldService.save(calculatedField)); + logEntityActionService.logEntityAction(tenantId, savedCalculatedField.getId(), savedCalculatedField, actionType, user); + return savedCalculatedField; + } catch (ThingsboardException e) { + logEntityActionService.logEntityAction(tenantId, emptyId(EntityType.CALCULATED_FIELD), calculatedField, actionType, user, e); + throw e; + } + } + + @Override + public CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user) { + return calculatedFieldService.findById(user.getTenantId(), calculatedFieldId); + } + + @Override + public PageData findAllByTenantIdAndEntityId(EntityId entityId, SecurityUser user, PageLink pageLink) { + TenantId tenantId = user.getTenantId(); + checkEntityExistence(tenantId, entityId); + return calculatedFieldService.findAllCalculatedFieldsByEntityId(tenantId, entityId, pageLink); + } + + @Override + @Transactional + public void delete(CalculatedField calculatedField, SecurityUser user) { + ActionType actionType = ActionType.DELETED; + TenantId tenantId = calculatedField.getTenantId(); + CalculatedFieldId calculatedFieldId = calculatedField.getId(); + try { + calculatedFieldService.deleteCalculatedField(tenantId, calculatedFieldId); + logEntityActionService.logEntityAction(tenantId, calculatedFieldId, calculatedField, actionType, user, calculatedFieldId.toString()); + } catch (Exception e) { + logEntityActionService.logEntityAction(tenantId, emptyId(EntityType.CALCULATED_FIELD), actionType, user, e, calculatedFieldId.toString()); + throw e; + } + } + + private void checkForEntityChange(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { + if (!oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId())) { + throw new IllegalArgumentException("Changing the calculated field target entity after initialization is prohibited."); + } + } + + private void checkEntityExistence(TenantId tenantId, EntityId entityId) { + switch (entityId.getEntityType()) { + case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> Optional.ofNullable(entityService.fetchEntity(tenantId, entityId)) + .orElseThrow(() -> new IllegalArgumentException(entityId.getEntityType().getNormalName() + " with id [" + entityId.getId() + "] does not exist.")); + default -> throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java new file mode 100644 index 0000000000..1e04a14a08 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.service.security.model.SecurityUser; + +public interface TbCalculatedFieldService { + + CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException; + + CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user); + + PageData findAllByTenantIdAndEntityId(EntityId entityId, SecurityUser user, PageLink pageLink); + + void delete(CalculatedField calculatedField, SecurityUser user); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java new file mode 100644 index 0000000000..bce7250bf7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.dao.cf.CalculatedFieldService; + +@Component +@RequiredArgsConstructor +@Slf4j +public class CalculatedFieldsDeletionTaskProcessor extends HousekeeperTaskProcessor { + + private final CalculatedFieldService calculatedFieldService; + + @Override + public void process(HousekeeperTask task) throws Exception { + int deletedCount = calculatedFieldService.deleteAllCalculatedFieldsByEntityId(task.getTenantId(), task.getEntityId()); + log.debug("[{}][{}][{}] Deleted {} calculated fields", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_CALCULATED_FIELDS; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index b2cde03931..870ce3838b 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -69,6 +69,7 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.mobile.app.MobileApp; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.page.PageLink; @@ -98,9 +99,9 @@ import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.mobile.MobileAppDao; import org.thingsboard.server.dao.notification.NotificationSettingsService; import org.thingsboard.server.dao.notification.NotificationTargetService; -import org.thingsboard.server.dao.mobile.MobileAppDao; import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.settings.AdminSettingsService; @@ -308,7 +309,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { jwtSettingsService.saveJwtSettings(jwtSettings); } - List mobiles = mobileAppDao.findByTenantId(TenantId.SYS_TENANT_ID, null, new PageLink(Integer.MAX_VALUE,0)).getData(); + List mobiles = mobileAppDao.findByTenantId(TenantId.SYS_TENANT_ID, null, new PageLink(Integer.MAX_VALUE, 0)).getData(); if (CollectionUtils.isNotEmpty(mobiles)) { mobiles.stream() .filter(mobileApp -> !validateKeyLength(mobileApp.getAppSecret())) @@ -571,7 +572,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { private void save(DeviceId deviceId, String key, boolean value) { if (persistActivityToTelemetry) { - ListenableFuture saveFuture = tsService.save( + ListenableFuture saveFuture = tsService.save( TenantId.SYS_TENANT_ID, deviceId, Collections.singletonList(new BasicTsKvEntry(System.currentTimeMillis(), new BooleanDataEntry(key, value))), 0L); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java new file mode 100644 index 0000000000..e76a2be9be --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -0,0 +1,269 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldLinkedTelemetryMsg; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.queue.QueueConfig; +import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldCache; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import org.thingsboard.server.service.queue.processing.AbstractConsumerService; +import org.thingsboard.server.service.queue.processing.IdMsgPair; +import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; + +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.thingsboard.server.common.util.ProtoUtils.fromProto; + +@Service +@TbRuleEngineComponent +@Slf4j +public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerService implements TbCalculatedFieldConsumerService { + + @Value("${queue.calculated_fields.poll_interval:25}") + private long pollInterval; + @Value("${queue.calculated_fields.pack_processing_timeout:60000}") + private long packProcessingTimeout; + @Value("${queue.calculated_fields.pool_size:8}") + private int poolSize; + + private final TbRuleEngineQueueFactory queueFactory; + private final CalculatedFieldStateService stateService; + + private PartitionedQueueConsumerManager> eventConsumer; + + public DefaultTbCalculatedFieldConsumerService(TbRuleEngineQueueFactory tbQueueFactory, + ActorSystemContext actorContext, + TbDeviceProfileCache deviceProfileCache, + TbAssetProfileCache assetProfileCache, + TbTenantProfileCache tenantProfileCache, + TbApiUsageStateService apiUsageStateService, + PartitionService partitionService, + ApplicationEventPublisher eventPublisher, + JwtSettingsService jwtSettingsService, + CalculatedFieldCache calculatedFieldCache, + CalculatedFieldStateService stateService) { + super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, + eventPublisher, jwtSettingsService); + this.queueFactory = tbQueueFactory; + this.stateService = stateService; + } + + @PostConstruct + public void init() { + super.init("tb-cf"); + + this.eventConsumer = PartitionedQueueConsumerManager.>create() + .queueKey(QueueKey.CF) + .topic(partitionService.getTopic(QueueKey.CF)) + .pollInterval(pollInterval) + .msgPackProcessor(this::processMsgs) + .consumerCreator((config, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer()) + .consumerExecutor(consumersExecutor) + .scheduler(scheduler) + .taskExecutor(mgmtExecutor) + .build(); + stateService.init(eventConsumer); + } + + @PreDestroy + public void destroy() { + super.destroy(); + } + + @Override + protected void startConsumers() { + super.startConsumers(); + } + + @Override + protected void onTbApplicationEvent(PartitionChangeEvent event) { + var partitions = event.getCfPartitions(); + try { + stateService.restore(partitions); + // eventConsumer's partitions will be updated by stateService + + // Cleanup old entities after corresponding consumers are stopped. + // Any periodic tasks need to check that the entity is still managed by the current server before processing. + actorContext.tell(new CalculatedFieldPartitionChangeMsg(partitionsToBooleanIndexArray(partitions))); + } catch (Throwable t) { + log.error("Failed to process partition change event: {}", event, t); + } + } + + private boolean[] partitionsToBooleanIndexArray(Set partitions) { + boolean[] myPartitions = new boolean[partitionService.getTotalCalculatedFieldPartitions()]; + for (var tpi : partitions) { + tpi.getPartition().ifPresent(partition -> myPartitions[partition] = true); + } + return myPartitions; + } + + private void processMsgs(List> msgs, TbQueueConsumer> consumer, QueueConfig config) throws Exception { + List> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList(); + ConcurrentMap> pendingMap = orderedMsgList.stream().collect( + Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + TbPackProcessingContext> ctx = new TbPackProcessingContext<>( + processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); + PendingMsgHolder pendingMsgHolder = new PendingMsgHolder<>(); + Future packSubmitFuture = consumersExecutor.submit(() -> { + orderedMsgList.forEach((element) -> { + UUID id = element.getUuid(); + TbProtoQueueMsg msg = element.getMsg(); + log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); + TbCallback callback = new TbPackCallback<>(id, ctx); + try { + ToCalculatedFieldMsg toCfMsg = msg.getValue(); + pendingMsgHolder.setMsg(toCfMsg); + if (toCfMsg.hasTelemetryMsg()) { + log.trace("[{}] Forwarding regular telemetry message for processing {}", id, toCfMsg.getTelemetryMsg()); + forwardToActorSystem(toCfMsg.getTelemetryMsg(), callback); + } else if (toCfMsg.hasLinkedTelemetryMsg()) { + forwardToActorSystem(toCfMsg.getLinkedTelemetryMsg(), callback); + } else if (toCfMsg.hasComponentLifecycleMsg()) { + log.trace("[{}] Forwarding component lifecycle message for processing {}", id, toCfMsg.getComponentLifecycleMsg()); + forwardToActorSystem(toCfMsg.getComponentLifecycleMsg(), callback); + } + } catch (Throwable e) { + log.warn("[{}] Failed to process message: {}", id, msg, e); + callback.onFailure(e); + } + }); + }); + if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + if (!packSubmitFuture.isDone()) { + packSubmitFuture.cancel(true); + log.info("Timeout to process message: {}", pendingMsgHolder.getMsg()); + } + if (log.isDebugEnabled()) { + ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); + } + ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); + } + consumer.commit(); + } + + @Override + protected ServiceType getServiceType() { + return ServiceType.TB_RULE_ENGINE; + } + + @Override + protected long getNotificationPollDuration() { + return pollInterval; + } + + @Override + protected long getNotificationPackProcessingTimeout() { + return packProcessingTimeout; + } + + @Override + protected int getMgmtThreadPoolSize() { + return Math.max(Runtime.getRuntime().availableProcessors(), 4); + } + + @Override + protected TbQueueConsumer> createNotificationsConsumer() { + return queueFactory.createToCalculatedFieldNotificationsMsgConsumer(); + } + + @Override + protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { + ToCalculatedFieldNotificationMsg toCfNotification = msg.getValue(); + if (toCfNotification.hasComponentLifecycleMsg()) { + // from upstream (maybe removed since we don't need to init state for each partition) + log.trace("[{}] Forwarding component lifecycle message for processing {}", id, toCfNotification.getComponentLifecycleMsg()); + forwardToActorSystem(toCfNotification.getComponentLifecycleMsg(), callback); + } else if (toCfNotification.hasLinkedTelemetryMsg()) { + forwardToActorSystem(toCfNotification.getLinkedTelemetryMsg(), callback); + } + } + + private void forwardToActorSystem(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { + var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); + var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + actorContext.tell(new CalculatedFieldTelemetryMsg(tenantId, entityId, msg, callback)); + } + + private void forwardToActorSystem(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { + var msg = linkedMsg.getMsg(); + var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); + var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + actorContext.tell(new CalculatedFieldLinkedTelemetryMsg(tenantId, entityId, linkedMsg, callback)); + } + + private void forwardToActorSystem(ComponentLifecycleMsgProto proto, TbCallback callback) { + var msg = fromProto(proto); + actorContext.tell(new CalculatedFieldEntityLifecycleMsg(msg.getTenantId(), msg, callback)); + } + + private TenantId toTenantId(long tenantIdMSB, long tenantIdLSB) { + return TenantId.fromUUID(new UUID(tenantIdMSB, tenantIdLSB)); + } + + @Override + protected void stopConsumers() { + super.stopConsumers(); + eventConsumer.stop(); + eventConsumer.awaitStop(); + stateService.stop(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index af298a9b87..800578ace4 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AssetId; @@ -77,6 +78,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.QueueDeleteMsg; import org.thingsboard.server.gen.transport.TransportProtos.QueueUpdateMsg; import org.thingsboard.server.gen.transport.TransportProtos.ResourceDeleteMsg; import org.thingsboard.server.gen.transport.TransportProtos.ResourceUpdateMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; @@ -91,8 +94,10 @@ import org.thingsboard.server.queue.common.MultipleTbQueueCallbackWrapper; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbRuleEngineProducerService; import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -141,6 +146,10 @@ public class DefaultTbClusterService implements TbClusterService { @Lazy private OtaPackageStateService otaPackageStateService; + @Autowired + @Lazy + private CalculatedFieldProcessingService calculatedFieldProcessingService; + private final TopicService topicService; private final TbDeviceProfileCache deviceProfileCache; private final TbAssetProfileCache assetProfileCache; @@ -182,6 +191,19 @@ public class DefaultTbClusterService implements TbClusterService { } } + @Override + public void broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg toCfMsg, TbQueueCallback callback) { + UUID msgId = UUID.randomUUID(); + TbQueueProducer> toCfProducer = producerProvider.getCalculatedFieldsNotificationsMsgProducer(); + Set tbReServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); + MultipleTbQueueCallbackWrapper callbackWrapper = new MultipleTbQueueCallbackWrapper(tbReServices.size(), callback); + for (String serviceId : tbReServices) { + TopicPartitionInfo tpi = topicService.getCalculatedFieldNotificationsTopic(serviceId); + toCfProducer.send(tpi, new TbProtoQueueMsg<>(msgId, toCfMsg), callbackWrapper); + toRuleEngineNfs.incrementAndGet(); + } + } + @Override public void pushMsgToVersionControl(TenantId tenantId, ToVersionControlServiceMsg msg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_VC_EXECUTOR, TenantId.SYS_TENANT_ID, tenantId); @@ -334,6 +356,26 @@ public class DefaultTbClusterService implements TbClusterService { toTransportNfs.incrementAndGet(); } + @Override + public void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF, entityId); + pushMsgToCalculatedFields(tpi, UUID.randomUUID(), msg, callback); + } + + @Override + public void pushMsgToCalculatedFields(TopicPartitionInfo tpi, UUID msgId, ToCalculatedFieldMsg msg, TbQueueCallback callback) { + log.trace("PUSHING msg: {} to:{}", msg, tpi); + producerProvider.getCalculatedFieldsMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); + toRuleEngineMsgs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS + } + + @Override + public void pushNotificationToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldNotificationMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF, entityId); + producerProvider.getCalculatedFieldsNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); + toRuleEngineNfs.incrementAndGet(); + } + @Override public void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); @@ -389,11 +431,19 @@ public class DefaultTbClusterService implements TbClusterService { public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) { DeviceId deviceId = device.getId(); gatewayNotificationsService.onDeviceDeleted(device); + handleCalculatedFieldEntityDeleted(tenantId, deviceId); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); } + @Override + public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { + AssetId assetId = asset.getId(); + handleCalculatedFieldEntityDeleted(tenantId, assetId); + broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); + } + @Override public void onDeviceAssignedToTenant(TenantId oldTenantId, Device device) { onDeviceDeleted(oldTenantId, device, null); @@ -553,7 +603,8 @@ public class DefaultTbClusterService implements TbClusterService { || entityType.equals(EntityType.API_USAGE_STATE) || (entityType.equals(EntityType.DEVICE) && msg.getEvent() == ComponentLifecycleEvent.UPDATED) || entityType.equals(EntityType.ENTITY_VIEW) - || entityType.equals(EntityType.NOTIFICATION_RULE)) { + || entityType.equals(EntityType.NOTIFICATION_RULE) + ) { TbQueueProducer> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); for (String serviceId : tbCoreServices) { @@ -604,21 +655,90 @@ public class DefaultTbClusterService implements TbClusterService { } @Override - public void onDeviceUpdated(Device device, Device old) { + public void onDeviceUpdated(Device entity, Device old) { var created = old == null; - broadcastEntityChangeToTransport(device.getTenantId(), device.getId(), device, null); + broadcastEntityChangeToTransport(entity.getTenantId(), entity.getId(), entity, null); if (old != null) { - boolean deviceNameChanged = !device.getName().equals(old.getName()); + boolean deviceNameChanged = !entity.getName().equals(old.getName()); if (deviceNameChanged) { - gatewayNotificationsService.onDeviceUpdated(device, old); + gatewayNotificationsService.onDeviceUpdated(entity, old); } - if (deviceNameChanged || !device.getType().equals(old.getType())) { - pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); + boolean deviceProfileChanged = !entity.getDeviceProfileId().equals(old.getDeviceProfileId()); + if (deviceProfileChanged) { + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(entity.getTenantId()) + .entityId(entity.getId()) + .event(ComponentLifecycleEvent.UPDATED) + .oldProfileId(old.getDeviceProfileId()) + .profileId(entity.getDeviceProfileId()) + .oldName(old.getName()) + .name(entity.getName()) + .build(); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + } + if (deviceNameChanged || deviceProfileChanged) { + pushMsgToCore(new DeviceNameOrTypeUpdateMsg(entity.getTenantId(), entity.getId(), entity.getName(), entity.getType()), null); } + } else { + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(entity.getTenantId()) + .entityId(entity.getId()) + .event(ComponentLifecycleEvent.CREATED) + .profileId(entity.getDeviceProfileId()) + .name(entity.getName()) + .build(); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } - broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), created, !created, false); - otaPackageStateService.update(device, old); + broadcastEntityStateChangeEvent(entity.getTenantId(), entity.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + sendDeviceStateServiceEvent(entity.getTenantId(), entity.getId(), created, !created, false); + otaPackageStateService.update(entity, old); + } + + @Override + public void onAssetUpdated(Asset entity, Asset old) { + var created = old == null; + if (old != null) { + boolean assetTypeChanged = !entity.getAssetProfileId().equals(old.getAssetProfileId()); + if (assetTypeChanged) { + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(entity.getTenantId()) + .entityId(entity.getId()) + .event(ComponentLifecycleEvent.UPDATED) + .oldProfileId(old.getAssetProfileId()) + .profileId(entity.getAssetProfileId()) + .oldName(old.getName()) + .name(entity.getName()) + .build(); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + } + } else { + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(entity.getTenantId()) + .entityId(entity.getId()) + .event(ComponentLifecycleEvent.CREATED) + .profileId(entity.getAssetProfileId()) + .name(entity.getName()) + .build(); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + } + broadcastEntityStateChangeEvent(entity.getTenantId(), entity.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + } + + @Override + public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, TbQueueCallback callback) { + var msg = toProto(new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), oldCalculatedField == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED)); + onCalculatedFieldLifecycleMsg(msg, callback); + } + + @Override + public void onCalculatedFieldDeleted(CalculatedField calculatedField, TbQueueCallback callback) { + var msg = toProto(new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), ComponentLifecycleEvent.DELETED)); + onCalculatedFieldLifecycleMsg(msg, callback); + } + + private void onCalculatedFieldLifecycleMsg(ComponentLifecycleMsgProto msg, TbQueueCallback callback) { + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(msg).build(), callback); + broadcastToCore(ToCoreNotificationMsg.newBuilder().setComponentLifecycle(msg).build()); } @Override @@ -748,4 +868,8 @@ public class DefaultTbClusterService implements TbClusterService { } } + private void handleCalculatedFieldEntityDeleted(TenantId tenantId, EntityId entityId) { + ComponentLifecycleMsg msg = new ComponentLifecycleMsg(tenantId, entityId, ComponentLifecycleEvent.DELETED); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 586902d542..a3003ba6ff 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -20,9 +20,6 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; -import lombok.Data; -import lombok.Getter; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; @@ -35,6 +32,7 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.JavaSerDesUtil; import org.thingsboard.server.common.data.alarm.AlarmInfo; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.LifecycleEvent; @@ -47,6 +45,7 @@ import org.thingsboard.server.common.data.queue.QueueConfig; import org.thingsboard.server.common.data.rpc.RpcError; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -78,6 +77,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceM import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.QueueKey; @@ -85,11 +85,11 @@ import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldCache; import org.thingsboard.server.service.notification.NotificationSchedulerService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; -import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; import org.thingsboard.server.service.queue.processing.IdMsgPair; import org.thingsboard.server.service.resource.TbImageService; @@ -147,9 +147,10 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig> mainConsumer; + private MainQueueConsumerManager, QueueConfig> mainConsumer; private QueueConsumerManager> usageStatsConsumer; private QueueConsumerManager> firmwareStatesConsumer; @@ -175,8 +176,10 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig>builder() + this.mainConsumer = MainQueueConsumerManager., QueueConfig>builder() .queueKey(new QueueKey(ServiceType.TB_CORE)) - .config(CoreQueueConfig.of(consumerPerPartition, (int) pollInterval)) + .config(QueueConfig.of(consumerPerPartition, pollInterval)) .msgPackProcessor(this::processMsgs) .consumerCreator((config, partitionId) -> queueFactory.createToCoreMsgConsumer()) .consumerExecutor(consumersExecutor) @@ -251,14 +255,14 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> msgs, TbQueueConsumer> consumer, CoreQueueConfig config) throws Exception { + private void processMsgs(List> msgs, TbQueueConsumer> consumer, QueueConfig config) throws Exception { List> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList(); ConcurrentMap> pendingMap = orderedMsgList.stream().collect( Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); TbPackProcessingContext> ctx = new TbPackProcessingContext<>( processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); - PendingMsgHolder pendingMsgHolder = new PendingMsgHolder(); + PendingMsgHolder pendingMsgHolder = new PendingMsgHolder<>(); Future packSubmitFuture = consumersExecutor.submit(() -> { orderedMsgList.forEach((element) -> { UUID id = element.getUuid(); @@ -267,7 +271,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService(id, ctx); try { ToCoreMsg toCoreMsg = msg.getValue(); - pendingMsgHolder.setToCoreMsg(toCoreMsg); + pendingMsgHolder.setMsg(toCoreMsg); if (toCoreMsg.hasToSubscriptionMgrMsg()) { log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg()); forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback); @@ -289,6 +293,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); @@ -329,12 +335,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = deviceActivityEventsExecutor.submit(() -> stateService.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, deviceInactivityTimeoutUpdateMsg.getInactivityTimeout())); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process device inactivity timeout update message for device [{}]", tenantId.getId(), deviceId.getId(), t); + callback.onFailure(t); + }); + } + private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); @@ -730,10 +757,4 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, EdgeQueueConfig> mainConsumer; + private MainQueueConsumerManager, QueueConfig> mainConsumer; public DefaultTbEdgeConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, StatsFactory statsFactory, EdgeContextComponent edgeCtx) { - super(actorContext, null, null, null, null, null, + super(actorContext, null, null, null, null, null, null, null, null); this.edgeCtx = edgeCtx; this.stats = new EdgeConsumerStats(statsFactory); @@ -102,9 +100,9 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService, EdgeQueueConfig>builder() + this.mainConsumer = MainQueueConsumerManager., QueueConfig>builder() .queueKey(new QueueKey(ServiceType.TB_CORE).withQueueName(DataConstants.EDGE_QUEUE_NAME)) - .config(EdgeQueueConfig.of(consumerPerPartition, pollInterval)) + .config(QueueConfig.of(consumerPerPartition, pollInterval)) .msgPackProcessor(this::processMsgs) .consumerCreator((config, partitionId) -> queueFactory.createEdgeMsgConsumer()) .consumerExecutor(consumersExecutor) @@ -130,14 +128,14 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService> msgs, TbQueueConsumer> consumer, EdgeQueueConfig edgeQueueConfig) throws InterruptedException { + private void processMsgs(List> msgs, TbQueueConsumer> consumer, QueueConfig edgeQueueConfig) throws InterruptedException { List> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList(); ConcurrentMap> pendingMap = orderedMsgList.stream().collect( Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); CountDownLatch processingTimeoutLatch = new CountDownLatch(1); TbPackProcessingContext> ctx = new TbPackProcessingContext<>( processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); - PendingMsgHolder pendingMsgHolder = new PendingMsgHolder(); + PendingMsgHolder pendingMsgHolder = new PendingMsgHolder<>(); Future submitFuture = consumersExecutor.submit(() -> { orderedMsgList.forEach((element) -> { UUID id = element.getUuid(); @@ -145,7 +143,7 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService(id, ctx); try { ToEdgeMsg toEdgeMsg = msg.getValue(); - pendingMsgHolder.setToEdgeMsg(toEdgeMsg); + pendingMsgHolder.setMsg(toEdgeMsg); if (toEdgeMsg.hasEdgeNotificationMsg()) { pushNotificationToEdge(toEdgeMsg.getEdgeNotificationMsg(), 0, packProcessingRetries, callback); } @@ -161,20 +159,13 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService log.warn("[{}] Failed to process message: {}", id, msg.getValue())); } consumer.commit(); } - private static class PendingMsgHolder { - @Getter - @Setter - private volatile ToEdgeMsg toEdgeMsg; - } - @Override protected ServiceType getServiceType() { return ServiceType.TB_CORE; @@ -294,10 +285,4 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService { + event.getNewPartitions().forEach((queueKey, partitions) -> { + if (CollectionsUtil.isOneOf(queueKey, QueueKey.CF, QueueKey.CF_STATES)) { + return; + } if (partitionService.isManagedByCurrentService(queueKey.getTenantId())) { var consumer = getConsumer(queueKey).orElseGet(() -> { Queue config = queueService.findQueueByTenantIdAndName(queueKey.getTenantId(), queueKey.getQueueName()); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/PendingMsgHolder.java b/application/src/main/java/org/thingsboard/server/service/queue/PendingMsgHolder.java new file mode 100644 index 0000000000..8f9cb3d092 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/PendingMsgHolder.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.Getter; +import lombok.Setter; + +public class PendingMsgHolder { + @Getter @Setter + private volatile T msg; +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java new file mode 100644 index 0000000000..8c7a459fab --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; + +public interface TbCalculatedFieldConsumerService extends ApplicationListener { + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java index 728f18bfff..46a42284b4 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java @@ -40,6 +40,7 @@ public class TbCoreConsumerStats { public static final String DEVICE_ACTIVITIES = "deviceActivity"; public static final String DEVICE_DISCONNECTS = "deviceDisconnect"; public static final String DEVICE_INACTIVITIES = "deviceInactivity"; + public static final String DEVICE_INACTIVITY_TIMEOUT_UPDATES = "deviceInactivityTimeoutUpdate"; public static final String TO_CORE_NF_OTHER = "coreNfOther"; // normally, there is no messages when codebase is fine public static final String TO_CORE_NF_COMPONENT_LIFECYCLE = "coreNfCompLfcl"; @@ -65,6 +66,7 @@ public class TbCoreConsumerStats { private final StatsCounter deviceActivitiesCounter; private final StatsCounter deviceDisconnectsCounter; private final StatsCounter deviceInactivitiesCounter; + private final StatsCounter deviceInactivityTimeoutUpdatesCounter; private final StatsCounter toCoreNfOtherCounter; private final StatsCounter toCoreNfComponentLifecycleCounter; @@ -95,6 +97,7 @@ public class TbCoreConsumerStats { this.deviceActivitiesCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_ACTIVITIES)); this.deviceDisconnectsCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_DISCONNECTS)); this.deviceInactivitiesCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_INACTIVITIES)); + this.deviceInactivityTimeoutUpdatesCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_INACTIVITY_TIMEOUT_UPDATES)); // Core notification counters this.toCoreNfOtherCounter = register(statsFactory.createStatsCounter(statsKey, TO_CORE_NF_OTHER)); @@ -163,6 +166,11 @@ public class TbCoreConsumerStats { deviceInactivitiesCounter.increment(); } + public void log(TransportProtos.DeviceInactivityTimeoutUpdateProto msg) { + totalCounter.increment(); + deviceInactivityTimeoutUpdatesCounter.increment(); + } + public void log(TransportProtos.SubscriptionMgrMsgProto msg) { totalCounter.increment(); subscriptionMsgCounter.increment(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java index 4c8edefd36..93112ca0f3 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.queue; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -23,9 +24,11 @@ import java.util.UUID; @Slf4j public class TbPackCallback implements TbCallback { private final TbPackProcessingContext ctx; + @Getter private final UUID id; public TbPackCallback(UUID id, TbPackProcessingContext ctx) { + log.trace("[{}] CALLBACK CREATED", id); this.id = id; this.ctx = ctx; } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 604f76aac1..d12bd896ff 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -25,6 +25,7 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -43,6 +44,7 @@ import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldCache; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.queue.TbPackCallback; @@ -68,6 +70,7 @@ public abstract class AbstractConsumerService partitions; - private boolean drainQueue; - - public static TbQueueConsumerManagerTask delete(boolean drainQueue) { - return new TbQueueConsumerManagerTask(QueueEvent.DELETE, null, null, drainQueue); - } - - public static TbQueueConsumerManagerTask configUpdate(QueueConfig config) { - return new TbQueueConsumerManagerTask(QueueEvent.CONFIG_UPDATE, config, null, false); - } - - public static TbQueueConsumerManagerTask partitionChange(Set partitions) { - return new TbQueueConsumerManagerTask(QueueEvent.PARTITION_CHANGE, null, partitions, false); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index febbf4fdd4..c7f7f600a7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -33,11 +33,14 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; +import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask; +import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.DeleteQueueTask; +import org.thingsboard.server.queue.common.consumer.TbQueueConsumerTask; import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.service.queue.TbMsgPackCallback; import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; -import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; @@ -69,19 +72,19 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager findLwM2mObject(TenantId tenantId, String sortOrder, diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java index 3b5c257155..947095ab3a 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java @@ -19,6 +19,6 @@ public enum Operation { ALL, CREATE, READ, WRITE, DELETE, ASSIGN_TO_CUSTOMER, UNASSIGN_FROM_CUSTOMER, RPC_CALL, READ_CREDENTIALS, WRITE_CREDENTIALS, READ_ATTRIBUTES, WRITE_ATTRIBUTES, READ_TELEMETRY, WRITE_TELEMETRY, CLAIM_DEVICES, - ASSIGN_TO_TENANT + ASSIGN_TO_TENANT, READ_CALCULATED_FIELD, WRITE_CALCULATED_FIELD } diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java index edbc6c16b9..9d7590f786 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java @@ -50,7 +50,9 @@ public enum Resource { VERSION_CONTROL, NOTIFICATION(EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_RULE), - MOBILE_APP_SETTINGS; + MOBILE_APP_SETTINGS, + CALCULATED_FIELD(EntityType.CALCULATED_FIELD); + private final Set entityTypes; Resource() { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index 76c0e3cf62..a072cf2738 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.service.security.model.SecurityUser; -@Component(value="tenantAdminPermissions") +@Component(value = "tenantAdminPermissions") public class TenantAdminPermissions extends AbstractPermissions { public TenantAdminPermissions() { @@ -55,13 +55,13 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.OAUTH2_CONFIGURATION_TEMPLATE, new PermissionChecker.GenericPermissionChecker(Operation.READ)); put(Resource.MOBILE_APP, tenantEntityPermissionChecker); put(Resource.MOBILE_APP_BUNDLE, tenantEntityPermissionChecker); + put(Resource.CALCULATED_FIELD, tenantEntityPermissionChecker); } public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { @Override public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) { - if (!user.getTenantId().equals(entity.getTenantId())) { return false; } diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateManager.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateManager.java new file mode 100644 index 0000000000..26750e887e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateManager.java @@ -0,0 +1,180 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.state; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.rule.engine.api.DeviceStateManager; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.SimpleTbQueueCallback; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; + +import java.util.Optional; +import java.util.function.Consumer; +import java.util.function.Supplier; + +@Slf4j +@Service +@RequiredArgsConstructor +public class DefaultDeviceStateManager implements DeviceStateManager { + + private final TbServiceInfoProvider serviceInfoProvider; + private final PartitionService partitionService; + + private final Optional deviceStateService; + private final TbClusterService clusterService; + + @Override + public void onDeviceConnect(TenantId tenantId, DeviceId deviceId, long connectTime, TbCallback callback) { + forwardToDeviceStateService(tenantId, deviceId, + deviceStateService -> { + log.debug("[{}][{}] Forwarding device connect event to local service. Connect time: [{}].", tenantId.getId(), deviceId.getId(), connectTime); + deviceStateService.onDeviceConnect(tenantId, deviceId, connectTime); + }, + () -> { + log.debug("[{}][{}] Sending device connect message to core. Connect time: [{}].", tenantId.getId(), deviceId.getId(), connectTime); + var deviceConnectMsg = TransportProtos.DeviceConnectProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) + .setLastConnectTime(connectTime) + .build(); + return TransportProtos.ToCoreMsg.newBuilder() + .setDeviceConnectMsg(deviceConnectMsg) + .build(); + }, callback); + } + + @Override + public void onDeviceActivity(TenantId tenantId, DeviceId deviceId, long activityTime, TbCallback callback) { + forwardToDeviceStateService(tenantId, deviceId, + deviceStateService -> { + log.debug("[{}][{}] Forwarding device activity event to local service. Activity time: [{}].", tenantId.getId(), deviceId.getId(), activityTime); + deviceStateService.onDeviceActivity(tenantId, deviceId, activityTime); + }, + () -> { + log.debug("[{}][{}] Sending device activity message to core. Activity time: [{}].", tenantId.getId(), deviceId.getId(), activityTime); + var deviceActivityMsg = TransportProtos.DeviceActivityProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) + .setLastActivityTime(activityTime) + .build(); + return TransportProtos.ToCoreMsg.newBuilder() + .setDeviceActivityMsg(deviceActivityMsg) + .build(); + }, callback); + } + + @Override + public void onDeviceDisconnect(TenantId tenantId, DeviceId deviceId, long disconnectTime, TbCallback callback) { + forwardToDeviceStateService(tenantId, deviceId, + deviceStateService -> { + log.debug("[{}][{}] Forwarding device disconnect event to local service. Disconnect time: [{}].", tenantId.getId(), deviceId.getId(), disconnectTime); + deviceStateService.onDeviceDisconnect(tenantId, deviceId, disconnectTime); + }, + () -> { + log.debug("[{}][{}] Sending device disconnect message to core. Disconnect time: [{}].", tenantId.getId(), deviceId.getId(), disconnectTime); + var deviceDisconnectMsg = TransportProtos.DeviceDisconnectProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) + .setLastDisconnectTime(disconnectTime) + .build(); + return TransportProtos.ToCoreMsg.newBuilder() + .setDeviceDisconnectMsg(deviceDisconnectMsg) + .build(); + }, callback); + } + + @Override + public void onDeviceInactivity(TenantId tenantId, DeviceId deviceId, long inactivityTime, TbCallback callback) { + forwardToDeviceStateService(tenantId, deviceId, + deviceStateService -> { + log.debug("[{}][{}] Forwarding device inactivity event to local service. Inactivity time: [{}].", tenantId.getId(), deviceId.getId(), inactivityTime); + deviceStateService.onDeviceInactivity(tenantId, deviceId, inactivityTime); + }, + () -> { + log.debug("[{}][{}] Sending device inactivity message to core. Inactivity time: [{}].", tenantId.getId(), deviceId.getId(), inactivityTime); + var deviceInactivityMsg = TransportProtos.DeviceInactivityProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) + .setLastInactivityTime(inactivityTime) + .build(); + return TransportProtos.ToCoreMsg.newBuilder() + .setDeviceInactivityMsg(deviceInactivityMsg) + .build(); + }, callback); + } + + @Override + public void onDeviceInactivityTimeoutUpdate(TenantId tenantId, DeviceId deviceId, long inactivityTimeout, TbCallback callback) { + forwardToDeviceStateService(tenantId, deviceId, + deviceStateService -> { + log.debug("[{}][{}] Forwarding device inactivity timeout update to local service. Updated inactivity timeout: [{}].", tenantId.getId(), deviceId.getId(), inactivityTimeout); + deviceStateService.onDeviceInactivityTimeoutUpdate(tenantId, deviceId, inactivityTimeout); + }, + () -> { + log.debug("[{}][{}] Sending device inactivity timeout update message to core. Updated inactivity timeout: [{}].", tenantId.getId(), deviceId.getId(), inactivityTimeout); + var deviceInactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) + .setInactivityTimeout(inactivityTimeout) + .build(); + return TransportProtos.ToCoreMsg.newBuilder() + .setDeviceInactivityTimeoutUpdateMsg(deviceInactivityTimeoutUpdateMsg) + .build(); + }, callback); + } + + private void forwardToDeviceStateService( + TenantId tenantId, DeviceId deviceId, + Consumer toDeviceStateService, + Supplier toCore, + TbCallback callback + ) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId); + if (serviceInfoProvider.isService(ServiceType.TB_CORE) && tpi.isMyPartition() && deviceStateService.isPresent()) { + try { + toDeviceStateService.accept(deviceStateService.get()); + } catch (Exception e) { + log.error("[{}][{}] Failed to process device connectivity event.", tenantId.getId(), deviceId.getId(), e); + callback.onFailure(e); + return; + } + callback.onSuccess(); + } else { + TransportProtos.ToCoreMsg toCoreMsg = toCore.get(); + clusterService.pushMsgToCore(tpi, deviceId.getId(), toCoreMsg, new SimpleTbQueueCallback(__ -> callback.onSuccess(), callback::onFailure)); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 11819ce5d8..cc476d377d 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -96,6 +96,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.UUID; @@ -129,11 +130,10 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService PERSISTENT_TELEMETRY_KEYS = Arrays.asList( new EntityKey(EntityKeyType.TIME_SERIES, LAST_ACTIVITY_TIME), new EntityKey(EntityKeyType.TIME_SERIES, INACTIVITY_ALARM_TIME), - new EntityKey(EntityKeyType.TIME_SERIES, INACTIVITY_TIMEOUT), new EntityKey(EntityKeyType.TIME_SERIES, ACTIVITY_STATE), new EntityKey(EntityKeyType.TIME_SERIES, LAST_CONNECT_TIME), new EntityKey(EntityKeyType.TIME_SERIES, LAST_DISCONNECT_TIME), - new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, INACTIVITY_TIMEOUT)); + new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, INACTIVITY_TIMEOUT)); // inactivity timeout is always a server attribute, even when activity data is stored as time series private static final List PERSISTENT_ATTRIBUTE_KEYS = Arrays.asList( new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, LAST_ACTIVITY_TIME), @@ -143,8 +143,14 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService PERSISTENT_ATTRIBUTES = Arrays.asList(ACTIVITY_STATE, LAST_CONNECT_TIME, - LAST_DISCONNECT_TIME, LAST_ACTIVITY_TIME, INACTIVITY_ALARM_TIME, INACTIVITY_TIMEOUT); + public static final Set ACTIVITY_KEYS_WITHOUT_INACTIVITY_TIMEOUT = Set.of( + ACTIVITY_STATE, LAST_CONNECT_TIME, LAST_DISCONNECT_TIME, LAST_ACTIVITY_TIME, INACTIVITY_ALARM_TIME + ); + + public static final Set ACTIVITY_KEYS_WITH_INACTIVITY_TIMEOUT = Set.of( + ACTIVITY_STATE, LAST_CONNECT_TIME, LAST_DISCONNECT_TIME, LAST_ACTIVITY_TIME, INACTIVITY_ALARM_TIME, INACTIVITY_TIMEOUT + ); + private static final List PERSISTENT_ENTITY_FIELDS = Arrays.asList( new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "type"), @@ -251,7 +257,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService= deviceState.getLastActivityTime()) { deviceState.setLastInactivityAlarmTime(0L); - save(deviceId, INACTIVITY_ALARM_TIME, 0L); + save(state.getTenantId(), deviceId, INACTIVITY_ALARM_TIME, 0L); } } } @@ -583,7 +589,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService fetchDeviceState(Device device) { ListenableFuture future; if (persistToTelemetry) { - ListenableFuture> tsData = tsService.findLatest(TenantId.SYS_TENANT_ID, device.getId(), PERSISTENT_ATTRIBUTES); - future = Futures.transform(tsData, extractDeviceStateData(device), MoreExecutors.directExecutor()); + ListenableFuture> timeseriesActivityDataFuture = tsService.findLatest(TenantId.SYS_TENANT_ID, device.getId(), ACTIVITY_KEYS_WITHOUT_INACTIVITY_TIMEOUT); + ListenableFuture> inactivityTimeoutAttributeFuture = attributesService.find( + TenantId.SYS_TENANT_ID, device.getId(), AttributeScope.SERVER_SCOPE, INACTIVITY_TIMEOUT + ); + + ListenableFuture> fullActivityDataFuture = Futures.whenAllSucceed(timeseriesActivityDataFuture, inactivityTimeoutAttributeFuture).call(() -> { + List activityTimeseries = Futures.getDone(timeseriesActivityDataFuture); + Optional inactivityTimeoutAttribute = Futures.getDone(inactivityTimeoutAttributeFuture); + + if (inactivityTimeoutAttribute.isPresent()) { + List result = new ArrayList<>(activityTimeseries.size() + 1); + result.addAll(activityTimeseries); + result.add(inactivityTimeoutAttribute.get()); + return result; + } else { + return activityTimeseries; + } + }, deviceStateCallbackExecutor); + + future = Futures.transform(fullActivityDataFuture, extractDeviceStateData(device), MoreExecutors.directExecutor()); } else { - ListenableFuture> attrData = attributesService.find(TenantId.SYS_TENANT_ID, device.getId(), AttributeScope.SERVER_SCOPE, PERSISTENT_ATTRIBUTES); - future = Futures.transform(attrData, extractDeviceStateData(device), MoreExecutors.directExecutor()); + ListenableFuture> attributesActivityDataFuture = attributesService.find( + TenantId.SYS_TENANT_ID, device.getId(), AttributeScope.SERVER_SCOPE, ACTIVITY_KEYS_WITH_INACTIVITY_TIMEOUT + ); + future = Futures.transform(attributesActivityDataFuture, extractDeviceStateData(device), MoreExecutors.directExecutor()); } - return transformInactivityTimeout(future); + return future; } - private ListenableFuture transformInactivityTimeout(ListenableFuture future) { - return Futures.transformAsync(future, deviceStateData -> { - if (!persistToTelemetry || deviceStateData.getState().getInactivityTimeout() != defaultInactivityTimeoutMs) { - return future; //fail fast - } - var attributesFuture = attributesService.find(TenantId.SYS_TENANT_ID, deviceStateData.getDeviceId(), AttributeScope.SERVER_SCOPE, INACTIVITY_TIMEOUT); - return Futures.transform(attributesFuture, attributes -> { - attributes.flatMap(KvEntry::getLongValue).ifPresent((inactivityTimeout) -> { - if (inactivityTimeout > 0) { - deviceStateData.getState().setInactivityTimeout(inactivityTimeout); - } - }); - return deviceStateData; - }, MoreExecutors.directExecutor()); - }, deviceStateCallbackExecutor); - } - - private Function, DeviceStateData> extractDeviceStateData(Device device) { + private Function, DeviceStateData> extractDeviceStateData(Device device) { return new Function<>() { @Nonnull @Override - public DeviceStateData apply(@Nullable List data) { + public DeviceStateData apply(@Nullable List data) { try { long lastActivityTime = getEntryValue(data, LAST_ACTIVITY_TIME, 0L); long inactivityAlarmTime = getEntryValue(data, INACTIVITY_ALARM_TIME, 0L); @@ -690,7 +698,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService 0 ? inactivityTimeout : defaultInactivityTimeoutMs) .build(); TbMsgMetaData md = new TbMsgMetaData(); md.putValue("deviceName", device.getName()); @@ -761,12 +769,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService deviceStateService; - private final TbClusterService clusterService; - - public DefaultRuleEngineDeviceStateManager( - TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService, - Optional deviceStateServiceOptional, TbClusterService clusterService - ) { - this.serviceInfoProvider = serviceInfoProvider; - this.partitionService = partitionService; - this.deviceStateService = deviceStateServiceOptional; - this.clusterService = clusterService; - } - - @Getter - private abstract static class ConnectivityEventInfo { - - private final TenantId tenantId; - private final DeviceId deviceId; - private final long eventTime; - - private ConnectivityEventInfo(TenantId tenantId, DeviceId deviceId, long eventTime) { - this.tenantId = tenantId; - this.deviceId = deviceId; - this.eventTime = eventTime; - } - - abstract void forwardToLocalService(); - - abstract TransportProtos.ToCoreMsg toQueueMsg(); - - } - - @Override - public void onDeviceConnect(TenantId tenantId, DeviceId deviceId, long connectTime, TbCallback callback) { - routeEvent(new ConnectivityEventInfo(tenantId, deviceId, connectTime) { - @Override - void forwardToLocalService() { - deviceStateService.ifPresent(service -> service.onDeviceConnect(tenantId, deviceId, connectTime)); - } - - @Override - TransportProtos.ToCoreMsg toQueueMsg() { - var deviceConnectMsg = TransportProtos.DeviceConnectProto.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) - .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) - .setLastConnectTime(connectTime) - .build(); - return TransportProtos.ToCoreMsg.newBuilder() - .setDeviceConnectMsg(deviceConnectMsg) - .build(); - } - }, callback); - } - - @Override - public void onDeviceActivity(TenantId tenantId, DeviceId deviceId, long activityTime, TbCallback callback) { - routeEvent(new ConnectivityEventInfo(tenantId, deviceId, activityTime) { - @Override - void forwardToLocalService() { - deviceStateService.ifPresent(service -> service.onDeviceActivity(tenantId, deviceId, activityTime)); - } - - @Override - TransportProtos.ToCoreMsg toQueueMsg() { - var deviceActivityMsg = TransportProtos.DeviceActivityProto.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) - .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) - .setLastActivityTime(activityTime) - .build(); - return TransportProtos.ToCoreMsg.newBuilder() - .setDeviceActivityMsg(deviceActivityMsg) - .build(); - } - }, callback); - } - - @Override - public void onDeviceDisconnect(TenantId tenantId, DeviceId deviceId, long disconnectTime, TbCallback callback) { - routeEvent(new ConnectivityEventInfo(tenantId, deviceId, disconnectTime) { - @Override - void forwardToLocalService() { - deviceStateService.ifPresent(service -> service.onDeviceDisconnect(tenantId, deviceId, disconnectTime)); - } - - @Override - TransportProtos.ToCoreMsg toQueueMsg() { - var deviceDisconnectMsg = TransportProtos.DeviceDisconnectProto.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) - .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) - .setLastDisconnectTime(disconnectTime) - .build(); - return TransportProtos.ToCoreMsg.newBuilder() - .setDeviceDisconnectMsg(deviceDisconnectMsg) - .build(); - } - }, callback); - } - - @Override - public void onDeviceInactivity(TenantId tenantId, DeviceId deviceId, long inactivityTime, TbCallback callback) { - routeEvent(new ConnectivityEventInfo(tenantId, deviceId, inactivityTime) { - @Override - void forwardToLocalService() { - deviceStateService.ifPresent(service -> service.onDeviceInactivity(tenantId, deviceId, inactivityTime)); - } - - @Override - TransportProtos.ToCoreMsg toQueueMsg() { - var deviceInactivityMsg = TransportProtos.DeviceInactivityProto.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) - .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) - .setLastInactivityTime(inactivityTime) - .build(); - return TransportProtos.ToCoreMsg.newBuilder() - .setDeviceInactivityMsg(deviceInactivityMsg) - .build(); - } - }, callback); - } - - private void routeEvent(ConnectivityEventInfo eventInfo, TbCallback callback) { - var tenantId = eventInfo.getTenantId(); - var deviceId = eventInfo.getDeviceId(); - long eventTime = eventInfo.getEventTime(); - - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId); - if (serviceInfoProvider.isService(ServiceType.TB_CORE) && tpi.isMyPartition() && deviceStateService.isPresent()) { - log.debug("[{}][{}] Forwarding device connectivity event to local service. Event time: [{}].", tenantId.getId(), deviceId.getId(), eventTime); - try { - eventInfo.forwardToLocalService(); - } catch (Exception e) { - log.error("[{}][{}] Failed to process device connectivity event. Event time: [{}].", tenantId.getId(), deviceId.getId(), eventTime, e); - callback.onFailure(e); - return; - } - callback.onSuccess(); - } else { - TransportProtos.ToCoreMsg msg = eventInfo.toQueueMsg(); - log.debug("[{}][{}] Sending device connectivity message to core. Event time: [{}].", tenantId.getId(), deviceId.getId(), eventTime); - clusterService.pushMsgToCore(tpi, UUID.randomUUID(), msg, new SimpleTbQueueCallback(__ -> callback.onSuccess(), callback::onFailure)); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java index 23adacfc2e..1fafb562c1 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java @@ -21,7 +21,6 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.id.DeviceId; @@ -49,8 +48,6 @@ import org.thingsboard.server.queue.discovery.event.OtherServiceShutdownEvent; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.state.DefaultDeviceStateService; -import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.ws.notification.sub.NotificationUpdate; import org.thingsboard.server.service.ws.notification.sub.NotificationsSubscriptionUpdate; @@ -76,7 +73,6 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueProducerProvider producerProvider; private final TbLocalSubscriptionService localSubscriptionService; - private final DeviceStateService deviceStateService; private final TbClusterService clusterService; private final SubscriptionSchedulerComponent scheduler; @@ -161,9 +157,6 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene @Override public void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts, TbCallback callback) { onTimeSeriesUpdate(entityId, ts); - if (entityId.getEntityType() == EntityType.DEVICE) { - updateDeviceInactivityTimeout(tenantId, entityId, ts); - } callback.onSuccess(); } @@ -171,13 +164,10 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene public void onTimeSeriesDelete(TenantId tenantId, EntityId entityId, List keys, TbCallback callback) { onTimeSeriesUpdate(entityId, keys.stream().map(key -> new BasicTsKvEntry(0, new StringDataEntry(key, ""))).collect(Collectors.toList())); - if (entityId.getEntityType() == EntityType.DEVICE) { - deleteDeviceInactivityTimeout(tenantId, entityId, keys); - } callback.onSuccess(); } - public void onTimeSeriesUpdate(EntityId entityId, List update) { + private void onTimeSeriesUpdate(EntityId entityId, List update) { getEntityUpdatesInfo(entityId).timeSeriesUpdateTs = System.currentTimeMillis(); TbEntityRemoteSubsInfo subInfo = entitySubscriptions.get(entityId); if (subInfo != null) { @@ -207,42 +197,27 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene @Override public void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, TbCallback callback) { - onAttributesUpdate(tenantId, entityId, scope, attributes, true, callback); - } - - @Override - public void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, TbCallback callback) { getEntityUpdatesInfo(entityId).attributesUpdateTs = System.currentTimeMillis(); processAttributesUpdate(entityId, scope, attributes); - if (entityId.getEntityType() == EntityType.DEVICE) { - if (TbAttributeSubscriptionScope.SERVER_SCOPE.name().equalsIgnoreCase(scope)) { - updateDeviceInactivityTimeout(tenantId, entityId, attributes); - } else if (TbAttributeSubscriptionScope.SHARED_SCOPE.name().equalsIgnoreCase(scope) && notifyDevice) { - clusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onUpdate(tenantId, - new DeviceId(entityId.getId()), DataConstants.SHARED_SCOPE, new ArrayList<>(attributes)) - , null); - } - } callback.onSuccess(); } + @Override + public void onAttributesDelete(TenantId tenantId, EntityId entityId, String scope, List keys, TbCallback callback) { + onAttributesDelete(tenantId, entityId, scope, keys, false, callback); + } + @Override public void onAttributesDelete(TenantId tenantId, EntityId entityId, String scope, List keys, boolean notifyDevice, TbCallback callback) { processAttributesUpdate(entityId, scope, keys.stream().map(key -> new BaseAttributeKvEntry(0, new StringDataEntry(key, ""))).collect(Collectors.toList())); - if (entityId.getEntityType() == EntityType.DEVICE) { - if (TbAttributeSubscriptionScope.SERVER_SCOPE.name().equalsIgnoreCase(scope) - || TbAttributeSubscriptionScope.ANY_SCOPE.name().equalsIgnoreCase(scope)) { - deleteDeviceInactivityTimeout(tenantId, entityId, keys); - } else if (TbAttributeSubscriptionScope.SHARED_SCOPE.name().equalsIgnoreCase(scope) && notifyDevice) { - clusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onDelete(tenantId, - new DeviceId(entityId.getId()), scope, keys), null); - } + if (entityId.getEntityType() == EntityType.DEVICE && TbAttributeSubscriptionScope.SHARED_SCOPE.name().equalsIgnoreCase(scope) && notifyDevice) { + clusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onDelete(tenantId, new DeviceId(entityId.getId()), scope, keys), null); } callback.onSuccess(); } - public void processAttributesUpdate(EntityId entityId, String scope, List update) { + private void processAttributesUpdate(EntityId entityId, String scope, List update) { TbEntityRemoteSubsInfo subInfo = entitySubscriptions.get(entityId); if (subInfo != null) { log.trace("[{}] Handling attributes update: {}", entityId, update); @@ -270,22 +245,6 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene } } - private void updateDeviceInactivityTimeout(TenantId tenantId, EntityId entityId, List kvEntries) { - for (KvEntry kvEntry : kvEntries) { - if (kvEntry.getKey().equals(DefaultDeviceStateService.INACTIVITY_TIMEOUT)) { - deviceStateService.onDeviceInactivityTimeoutUpdate(tenantId, new DeviceId(entityId.getId()), getLongValue(kvEntry)); - } - } - } - - private void deleteDeviceInactivityTimeout(TenantId tenantId, EntityId entityId, List keys) { - for (String key : keys) { - if (key.equals(DefaultDeviceStateService.INACTIVITY_TIMEOUT)) { - deviceStateService.onDeviceInactivityTimeoutUpdate(tenantId, new DeviceId(entityId.getId()), 0); - } - } - } - @Override public void onAlarmUpdate(TenantId tenantId, EntityId entityId, AlarmInfo alarm, TbCallback callback) { onAlarmSubUpdate(tenantId, entityId, alarm, false, callback); @@ -355,29 +314,6 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene } } - private static long getLongValue(KvEntry kve) { - switch (kve.getDataType()) { - case LONG: - return kve.getLongValue().orElse(0L); - case DOUBLE: - return kve.getDoubleValue().orElse(0.0).longValue(); - case STRING: - try { - return Long.parseLong(kve.getStrValue().orElse("0")); - } catch (NumberFormatException e) { - return 0L; - } - case JSON: - try { - return Long.parseLong(kve.getJsonValue().orElse("0")); - } catch (NumberFormatException e) { - return 0L; - } - default: - return 0L; - } - } - private static List getSubList(List ts, Set keys) { List update = null; for (T entry : ts) { diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java index 3fc314b632..9e9ca42e83 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java @@ -33,7 +33,7 @@ import org.springframework.stereotype.Service; import org.springframework.web.socket.CloseStatus; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.alarm.AlarmInfo; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; @@ -41,7 +41,6 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmDataQuery; import org.thingsboard.server.common.data.query.ComparisonTsValue; -import org.thingsboard.server.common.data.query.OriginatorAlarmFilter; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityKey; @@ -55,17 +54,16 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; -import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.ws.WebSocketService; import org.thingsboard.server.service.ws.WebSocketSessionRef; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AggHistoryCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AggKey; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AggTimeSeriesCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmCountCmd; +import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmCountUpdate; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmDataCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmDataUpdate; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmStatusCmd; -import org.thingsboard.server.service.ws.telemetry.cmd.v2.CmdUpdate; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityCountCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityDataCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityDataUpdate; @@ -74,7 +72,6 @@ import org.thingsboard.server.service.ws.telemetry.cmd.v2.GetTsCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.LatestValueCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.TimeSeriesCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.UnsubscribeCmd; -import org.thingsboard.server.service.ws.telemetry.sub.AlarmSubscriptionUpdate; import java.util.ArrayList; import java.util.Arrays; @@ -83,7 +80,6 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; @@ -430,13 +426,25 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc long start = System.currentTimeMillis(); ctx.fetchData(); long end = System.currentTimeMillis(); - stats.getAlarmQueryInvocationCnt().incrementAndGet(); - stats.getAlarmQueryTimeSpent().addAndGet(end - start); - TbAlarmCountSubCtx finalCtx = ctx; - ScheduledFuture task = scheduler.scheduleWithFixedDelay( - () -> refreshDynamicQuery(finalCtx), - dynamicPageLinkRefreshInterval, dynamicPageLinkRefreshInterval, TimeUnit.SECONDS); - finalCtx.setRefreshTask(task); + stats.getRegularQueryInvocationCnt().incrementAndGet(); + stats.getRegularQueryTimeSpent().addAndGet(end - start); + Set entitiesIds = ctx.getEntitiesIds(); + ctx.cancelTasks(); + ctx.clearAlarmSubscriptions(); + if (entitiesIds != null && entitiesIds.isEmpty()) { + AlarmCountUpdate update = new AlarmCountUpdate(cmd.getCmdId(), 0); + ctx.sendWsMsg(update); + } else { + ctx.doFetchAlarmCount(); + if (entitiesIds != null) { + ctx.createAlarmSubscriptions(); + } + TbAlarmCountSubCtx finalCtx = ctx; + ScheduledFuture task = scheduler.scheduleWithFixedDelay( + () -> refreshDynamicQuery(finalCtx), + dynamicPageLinkRefreshInterval, dynamicPageLinkRefreshInterval, TimeUnit.SECONDS); + finalCtx.setRefreshTask(task); + } } else { log.debug("[{}][{}] Received duplicate command: {}", session.getSessionId(), cmd.getCmdId(), cmd); } @@ -555,7 +563,7 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc private TbAlarmCountSubCtx createSubCtx(WebSocketSessionRef sessionRef, AlarmCountCmd cmd) { Map sessionSubs = subscriptionsBySessionId.computeIfAbsent(sessionRef.getSessionId(), k -> new ConcurrentHashMap<>()); TbAlarmCountSubCtx ctx = new TbAlarmCountSubCtx(serviceId, wsService, entityService, localSubscriptionService, - attributesService, stats, alarmService, sessionRef, cmd.getCmdId()); + attributesService, stats, alarmService, sessionRef, cmd.getCmdId(), maxEntitiesPerAlarmSubscription, maxAlarmQueriesPerRefreshInterval); if (cmd.getQuery() != null) { ctx.setAndResolveQuery(cmd.getQuery()); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java index d199f18b75..57d4fda5f8 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java @@ -39,8 +39,15 @@ public interface SubscriptionManagerService extends ApplicationListener attributes, TbCallback callback); - void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice, TbCallback callback); - + void onAttributesDelete(TenantId tenantId, EntityId entityId, String scope, List keys, TbCallback empty); + + /** + * This method is retained solely for backwards compatibility, specifically to handle + * legacy proto messages that include the notifyDevice field. + * + * @deprecated as of 4.0, this method will be removed in future releases. + */ + @Deprecated(forRemoval = true, since = "4.0") void onAttributesDelete(TenantId tenantId, EntityId entityId, String scope, List keys, boolean notifyDevice, TbCallback empty); void onTimeSeriesDelete(TenantId tenantId, EntityId entityId, List keys, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java index ae3c2e0ccd..6689c11c42 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java @@ -19,49 +19,152 @@ import lombok.Getter; import lombok.Setter; import lombok.ToString; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmCountQuery; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.service.ws.WebSocketService; import org.thingsboard.server.service.ws.WebSocketSessionRef; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmCountUpdate; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + @Slf4j @ToString(callSuper = true) public class TbAlarmCountSubCtx extends TbAbstractEntityQuerySubCtx { private final AlarmService alarmService; + protected final Map subToEntityIdMap; + + @Getter + private LinkedHashSet entitiesIds; + + private final int maxEntitiesPerAlarmSubscription; + + private final int maxAlarmQueriesPerRefreshInterval; + @Getter @Setter private volatile int result; + @Getter + @Setter + private boolean tooManyEntities; + + private int alarmCountInvocationAttempts; + public TbAlarmCountSubCtx(String serviceId, WebSocketService wsService, EntityService entityService, TbLocalSubscriptionService localSubscriptionService, AttributesService attributesService, SubscriptionServiceStatistics stats, AlarmService alarmService, - WebSocketSessionRef sessionRef, int cmdId) { + WebSocketSessionRef sessionRef, int cmdId, int maxEntitiesPerAlarmSubscription, int maxAlarmQueriesPerRefreshInterval) { super(serviceId, wsService, entityService, localSubscriptionService, attributesService, stats, sessionRef, cmdId); this.alarmService = alarmService; + this.subToEntityIdMap = new ConcurrentHashMap<>(); + this.maxEntitiesPerAlarmSubscription = maxEntitiesPerAlarmSubscription; + this.maxAlarmQueriesPerRefreshInterval = maxAlarmQueriesPerRefreshInterval; + this.entitiesIds = null; + } + + @Override + public void clearSubscriptions() { + clearAlarmSubscriptions(); } @Override public void fetchData() { - result = (int) alarmService.countAlarmsByQuery(getTenantId(), getCustomerId(), query); - sendWsMsg(new AlarmCountUpdate(cmdId, result)); + resetInvocationCounter(); + if (query.getEntityFilter() != null) { + entitiesIds = new LinkedHashSet<>(); + log.trace("[{}] Fetching data: {}", cmdId, alarmCountInvocationAttempts); + PageData data = entityService.findEntityDataByQuery(getTenantId(), getCustomerId(), buildEntityDataQuery()); + entitiesIds.clear(); + tooManyEntities = data.hasNext(); + for (EntityData entityData : data.getData()) { + entitiesIds.add(entityData.getEntityId()); + } + } } @Override protected void update() { - int newCount = (int) alarmService.countAlarmsByQuery(getTenantId(), getCustomerId(), query); - if (newCount != result) { - result = newCount; - sendWsMsg(new AlarmCountUpdate(cmdId, result)); - } + resetInvocationCounter(); + fetchAlarmCount(); } @Override public boolean isDynamic() { return true; } + + public void fetchAlarmCount() { + alarmCountInvocationAttempts++; + log.trace("[{}] Fetching alarms: {}", cmdId, alarmCountInvocationAttempts); + if (alarmCountInvocationAttempts <= maxAlarmQueriesPerRefreshInterval) { + int newCount = (int) alarmService.countAlarmsByQuery(getTenantId(), getCustomerId(), query, entitiesIds); + if (newCount != result) { + result = newCount; + sendWsMsg(new AlarmCountUpdate(cmdId, result)); + } + } else { + log.trace("[{}] Ignore alarm count fetch due to rate limit: [{}] of maximum [{}]", cmdId, alarmCountInvocationAttempts, maxAlarmQueriesPerRefreshInterval); + } + } + + public void doFetchAlarmCount() { + result = (int) alarmService.countAlarmsByQuery(getTenantId(), getCustomerId(), query, entitiesIds); + sendWsMsg(new AlarmCountUpdate(cmdId, result)); + } + + private EntityDataQuery buildEntityDataQuery() { + EntityDataPageLink edpl = new EntityDataPageLink(maxEntitiesPerAlarmSubscription, 0, null, + new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, ModelConstants.CREATED_TIME_PROPERTY))); + return new EntityDataQuery(query.getEntityFilter(), edpl, null, null, query.getKeyFilters()); + } + + private void resetInvocationCounter() { + alarmCountInvocationAttempts = 0; + } + + public void createAlarmSubscriptions() { + for (EntityId entityId : entitiesIds) { + createAlarmSubscriptionForEntity(entityId); + } + } + + private void createAlarmSubscriptionForEntity(EntityId entityId) { + int subIdx = sessionRef.getSessionSubIdSeq().incrementAndGet(); + subToEntityIdMap.put(subIdx, entityId); + log.trace("[{}][{}][{}] Creating alarms subscription for [{}] ", serviceId, cmdId, subIdx, entityId); + TbAlarmsSubscription subscription = TbAlarmsSubscription.builder() + .serviceId(serviceId) + .sessionId(sessionRef.getSessionId()) + .subscriptionId(subIdx) + .tenantId(sessionRef.getSecurityCtx().getTenantId()) + .entityId(entityId) + .updateProcessor((sub, update) -> fetchAlarmCount()) + .build(); + localSubscriptionService.addSubscription(subscription, sessionRef); + } + + public void clearAlarmSubscriptions() { + if (subToEntityIdMap != null) { + for (Integer subId : subToEntityIdMap.keySet()) { + localSubscriptionService.cancelSubscription(getTenantId(), getSessionId(), subId); + } + subToEntityIdMap.clear(); + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java index f6ce2a6a6b..f6b4067543 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java @@ -39,6 +39,7 @@ import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.sql.query.EntityKeyMapping; import org.thingsboard.server.service.ws.WebSocketService; import org.thingsboard.server.service.ws.WebSocketSessionRef; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmDataUpdate; @@ -359,7 +360,7 @@ public class TbAlarmDataSubCtx extends TbAbstractDataSubCtx { EntityDataSortOrder sortOrder = query.getPageLink().getSortOrder(); EntityDataSortOrder entitiesSortOrder; if (sortOrder == null || sortOrder.getKey().getType().equals(EntityKeyType.ALARM_FIELD)) { - entitiesSortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, ModelConstants.CREATED_TIME_PROPERTY)); + entitiesSortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, EntityKeyMapping.CREATED_TIME)); } else { entitiesSortOrder = sortOrder; } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java index 68c0f52f71..1d5e85cc22 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbSubscriptionUtils.java @@ -209,7 +209,7 @@ public class TbSubscriptionUtils { return ToCoreMsg.newBuilder().setToSubscriptionMgrMsg(msgBuilder.build()).build(); } - public static ToCoreMsg toAttributesDeleteProto(TenantId tenantId, EntityId entityId, String scope, List keys, boolean notifyDevice) { + public static ToCoreMsg toAttributesDeleteProto(TenantId tenantId, EntityId entityId, String scope, List keys) { TbAttributeDeleteProto.Builder builder = TbAttributeDeleteProto.newBuilder(); builder.setEntityType(entityId.getEntityType().name()); builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); @@ -218,7 +218,6 @@ public class TbSubscriptionUtils { builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); builder.setScope(scope); builder.addAllKeys(keys); - builder.setNotifyDevice(notifyDevice); SubscriptionMgrMsgProto.Builder msgBuilder = SubscriptionMgrMsgProto.newBuilder(); msgBuilder.setAttrDelete(builder); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 178dde33e2..06fe7f4036 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.util.ThrowingRunnable; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -61,6 +62,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS private final Map> importServices = new HashMap<>(); private final RelationService relationService; + private final CalculatedFieldService calculatedFieldService; private final RateLimitService rateLimitService; private final TbLogEntityActionService logEntityActionService; @@ -72,7 +74,6 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_RULE ); - @Override public , I extends EntityId> EntityExportData exportEntity(EntitiesExportCtx ctx, I entityId) throws ThingsboardException { if (!rateLimitService.checkRateLimit(LimitedApi.ENTITY_EXPORT, ctx.getTenantId())) { @@ -129,13 +130,11 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS } } - @Override public Comparator getEntityTypeComparatorForImport() { return Comparator.comparing(SUPPORTED_ENTITY_TYPES::indexOf); } - @SuppressWarnings("unchecked") private , D extends EntityExportData> EntityExportService getExportService(EntityType entityType) { EntityExportService exportService = exportServices.get(entityType); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java index 2b7e7d3593..b77447df8d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java @@ -32,7 +32,6 @@ public interface EntitiesExportImportService { , I extends EntityId> EntityImportResult importEntity(EntitiesImportCtx ctx, EntityExportData exportData) throws ThingsboardException; - void saveReferencesAndRelations(EntitiesImportCtx ctx) throws ThingsboardException; Comparator getEntityTypeComparatorForImport(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index b0bea68290..5c2fcc7dc6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasVersion; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -31,6 +32,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.sync.ie.AttributeExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; @@ -59,6 +61,8 @@ public class DefaultEntityExportService ctx, I entityId) throws ThingsboardException { @@ -98,6 +102,10 @@ public class DefaultEntityExportService> attributes = exportAttributes(ctx, entity); exportData.setAttributes(attributes); } + if (ctx.getSettings().isExportCalculatedFields()) { + List calculatedFields = exportCalculatedFields(ctx, entity.getId()); + exportData.setCalculatedFields(calculatedFields); + } } private List exportRelations(EntitiesExportCtx ctx, E entity) throws ThingsboardException { @@ -141,6 +149,19 @@ public class DefaultEntityExportService exportCalculatedFields(EntitiesExportCtx ctx, EntityId entityId) { + List calculatedFields = calculatedFieldService.findCalculatedFieldsByEntityId(ctx.getTenantId(), entityId); + calculatedFields.forEach(calculatedField -> { + calculatedField.setEntityId(getExternalIdOrElseInternal(ctx, entityId)); + calculatedField.getConfiguration().getArguments().values().forEach(argument -> { + if (argument.getRefEntityId() != null) { + argument.setRefEntityId(getExternalIdOrElseInternal(ctx, argument.getRefEntityId())); + } + }); + }); + return calculatedFields; + } + protected ID getExternalIdOrElseInternal(EntitiesExportCtx ctx, ID internalId) { if (internalId == null || internalId.isNullUid()) return internalId; var result = ctx.getExternalId(internalId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java index 00ce23b2e8..7cd4c3aca1 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java @@ -47,7 +47,11 @@ public class AssetImportService extends BaseEntityImportService exportData, IdProvider idProvider) { - return assetService.saveAsset(asset); + Asset savedAsset = assetService.saveAsset(asset); + if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) { + importCalculatedFields(ctx, savedAsset, exportData, idProvider); + } + return savedAsset; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java index 97c54957bf..32a0090a4a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetProfileImportService.java @@ -50,7 +50,11 @@ public class AssetProfileImportService extends BaseEntityImportService exportData, IdProvider idProvider) { - return assetProfileService.saveAssetProfile(assetProfile); + AssetProfile saved = assetProfileService.saveAssetProfile(assetProfile); + if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) { + importCalculatedFields(ctx, saved, exportData, idProvider); + } + return saved; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 7044b235a0..bfa95af83c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.HasDefaultOption; import org.thingsboard.server.common.data.HasVersion; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -50,6 +51,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.sync.ie.AttributeExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.action.EntityActionService; @@ -78,6 +80,8 @@ public abstract class BaseEntityImportService existing = calculatedFieldService.findCalculatedFieldsByEntityId(ctx.getTenantId(), savedEntity.getId()); + List fieldsToSave = exportData.getCalculatedFields().stream() + .peek(calculatedField -> { + calculatedField.setTenantId(ctx.getTenantId()); + calculatedField.setEntityId(savedEntity.getId()); + calculatedField.getConfiguration().getArguments().values().forEach(argument -> { + if (argument.getRefEntityId() != null) { + argument.setRefEntityId(idProvider.getInternalId(argument.getRefEntityId(), ctx.isFinalImportAttempt())); + } + }); + }).toList(); + + for (CalculatedField existingField : existing) { + boolean found = fieldsToSave.stream().anyMatch(importedField -> compareCalculatedFields(existingField, importedField)); + if (!found) { + calculatedFieldService.deleteCalculatedField(ctx.getTenantId(), existingField.getId()); + updated = true; + } + } + + for (CalculatedField calculatedField : fieldsToSave) { + boolean found = existing.stream().anyMatch(existingField -> compareCalculatedFields(existingField, calculatedField)); + if (!found) { + calculatedFieldService.save(calculatedField); + updated = true; + } + } + return updated; + } + + private boolean compareCalculatedFields(CalculatedField existingField, CalculatedField newField) { + CalculatedField oldCopy = new CalculatedField(existingField); + CalculatedField newCopy = new CalculatedField(newField); + oldCopy.setId(null); + newCopy.setId(null); + oldCopy.setVersion(null); + newCopy.setVersion(null); + oldCopy.setCreatedTime(0); + newCopy.setCreatedTime(0); + return oldCopy.equals(newCopy); + } + protected void onEntitySaved(User user, E savedEntity, E oldEntity) throws ThingsboardException { logEntityActionService.logEntityAction(user.getTenantId(), savedEntity.getId(), savedEntity, null, oldEntity == null ? ActionType.ADDED : ActionType.UPDATED, user); } - @SuppressWarnings("unchecked") protected E findExistingEntity(EntitiesImportCtx ctx, E entity, IdProvider idProvider) { return (E) Optional.ofNullable(entitiesService.findEntityByTenantIdAndExternalId(ctx.getTenantId(), entity.getId())) @@ -313,10 +364,10 @@ public abstract class BaseEntityImportService new MissingEntityException(externalId)); } - @SuppressWarnings("unchecked") @RequiredArgsConstructor protected class IdProvider { + private final EntitiesImportCtx ctx; private final EntityImportResult importResult; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java index 61b9839cbd..84e264efdd 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java @@ -64,13 +64,18 @@ public class DeviceImportService extends BaseEntityImportService exportData, IdProvider idProvider) { - return deviceProfileService.saveDeviceProfile(deviceProfile); + DeviceProfile saved = deviceProfileService.saveDeviceProfile(deviceProfile); + if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) { + importCalculatedFields(ctx, saved, exportData, idProvider); + } + return saved; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index eed18abe52..89b51ce84a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -94,7 +94,6 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.function.Function; -import java.util.stream.Collectors; import static com.google.common.util.concurrent.Futures.transform; import static org.thingsboard.server.common.data.sync.vc.VcUtils.checkBranchName; @@ -304,6 +303,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .updateRelations(config.isLoadRelations()) .saveAttributes(config.isLoadAttributes()) .saveCredentials(config.isLoadCredentials()) + .saveCalculatedFields(config.isLoadCalculatedFields()) .findExistingByName(false) .build()); ctx.setFinalImportAttempt(true); @@ -327,7 +327,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont var sw = TbStopWatch.create("before"); List entityTypes = request.getEntityTypes().keySet().stream() - .sorted(exportImportService.getEntityTypeComparatorForImport()).collect(Collectors.toList()); + .sorted(exportImportService.getEntityTypeComparatorForImport()).toList(); for (EntityType entityType : entityTypes) { log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType); sw.startNew("Entities " + entityType.name()); @@ -362,6 +362,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .updateRelations(config.isLoadRelations()) .saveAttributes(config.isLoadAttributes()) .saveCredentials(config.isLoadCredentials()) + .saveCalculatedFields(config.isLoadCalculatedFields()) .findExistingByName(config.isFindExistingEntityByName()) .build(); } @@ -471,7 +472,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } @Override - public ListenableFuture compareEntityDataToVersion(User user, EntityId entityId, String versionId) throws Exception { + public ListenableFuture compareEntityDataToVersion(User user, EntityId entityId, String versionId) { HasId entity = exportableEntitiesService.findEntityByTenantIdAndId(user.getTenantId(), entityId); if (!(entity instanceof ExportableEntity)) throw new IllegalArgumentException("Unsupported entity type"); @@ -484,6 +485,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .exportRelations(otherVersion.hasRelations()) .exportAttributes(otherVersion.hasAttributes()) .exportCredentials(otherVersion.hasCredentials()) + .exportCalculatedFields(otherVersion.hasCalculatedFields()) .build()); EntityExportData currentVersion; try { @@ -503,7 +505,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override - public ListenableFuture> listBranches(TenantId tenantId) throws Exception { + public ListenableFuture> listBranches(TenantId tenantId) { return gitServiceQueue.listBranches(tenantId); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java index 669dad347a..b3007c6738 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java @@ -69,6 +69,7 @@ public abstract class EntitiesExportCtx { .exportRelations(config.isSaveRelations()) .exportAttributes(config.isSaveAttributes()) .exportCredentials(config.isSaveCredentials()) + .exportCalculatedFields(config.isSaveCalculatedFields()) .build(); } @@ -85,4 +86,5 @@ public abstract class EntitiesExportCtx { log.debug("[{}][{}] Local cache put: {}", internalId.getEntityType(), internalId.getId(), externalId); externalIdMap.put(internalId, externalId != null ? externalId : internalId); } + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java index ab836fa13d..e24683cd5a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java @@ -91,6 +91,10 @@ public class EntitiesImportCtx { return getSettings().isSaveCredentials(); } + public boolean isSaveCalculatedFields() { + return getSettings().isSaveCalculatedFields(); + } + public EntityId getInternalId(EntityId externalId) { var result = externalToInternalIdMap.get(externalId); log.debug("[{}][{}] Local cache {} for id", externalId.getEntityType(), externalId.getId(), result != null ? "hit" : "miss"); @@ -140,5 +144,4 @@ public class EntitiesImportCtx { return notFoundIds.contains(externalId); } - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java index e0a5b40281..55c8a763b0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java @@ -39,6 +39,7 @@ public class EntityTypeExportCtx extends EntitiesExportCtx .exportRelations(config.isSaveRelations()) .exportAttributes(config.isSaveAttributes()) .exportCredentials(config.isSaveCredentials()) + .exportCalculatedFields(config.isSaveCalculatedFields()) .build(); this.overwrite = ObjectUtils.defaultIfNull(config.getSyncStrategy(), defaultSyncStrategy) == SyncStrategy.OVERWRITE; } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java index a5c631b29c..d6fa89a7a2 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java @@ -38,6 +38,7 @@ import org.thingsboard.server.service.subscription.SubscriptionManagerService; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Consumer; @@ -99,16 +100,27 @@ public abstract class AbstractSubscriptionService extends TbApplicationEventList } protected void addWsCallback(ListenableFuture saveFuture, Consumer callback) { - Futures.addCallback(saveFuture, new FutureCallback() { + addCallback(saveFuture, callback, wsCallBackExecutor); + } + + protected void addCallback(ListenableFuture saveFuture, Consumer callback, Executor executor) { + Futures.addCallback(saveFuture, new FutureCallback<>() { @Override public void onSuccess(@Nullable T result) { callback.accept(result); } @Override - public void onFailure(Throwable t) { - } - }, wsCallBackExecutor); + public void onFailure(Throwable t) {} + }, executor); + } + + protected static Consumer safeCallback(FutureCallback callback) { + if (callback != null) { + return callback::onFailure; + } else { + return throwable -> {}; + } } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 568d80787d..2302446b6b 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -31,29 +31,38 @@ import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.rule.engine.api.AttributesDeleteRequest; import org.thingsboard.rule.engine.api.AttributesSaveRequest; +import org.thingsboard.rule.engine.api.DeviceStateManager; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.rule.engine.DeviceAttributesEventNotificationMsg; import org.thingsboard.server.common.stats.TbApiUsageReportClient; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldQueueService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; +import org.thingsboard.server.service.state.DefaultDeviceStateService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import java.util.ArrayList; -import java.util.Comparator; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -63,6 +72,11 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Consumer; +import static java.util.Comparator.comparing; +import static java.util.Comparator.comparingLong; +import static java.util.Comparator.naturalOrder; +import static java.util.Comparator.nullsFirst; + /** * Created by ashvayka on 27.03.18. */ @@ -75,6 +89,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final TbEntityViewService tbEntityViewService; private final TbApiUsageReportClient apiUsageClient; private final TbApiUsageStateService apiUsageStateService; + private final CalculatedFieldQueueService calculatedFieldQueueService; + private final DeviceStateManager deviceStateManager; private ExecutorService tsCallBackExecutor; @@ -85,12 +101,16 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer TimeseriesService tsService, @Lazy TbEntityViewService tbEntityViewService, TbApiUsageReportClient apiUsageClient, - TbApiUsageStateService apiUsageStateService) { + TbApiUsageStateService apiUsageStateService, + CalculatedFieldQueueService calculatedFieldQueueService, + DeviceStateManager deviceStateManager) { this.attrService = attrService; this.tsService = tsService; this.tbEntityViewService = tbEntityViewService; this.apiUsageClient = apiUsageClient; this.apiUsageStateService = apiUsageStateService; + this.calculatedFieldQueueService = calculatedFieldQueueService; + this.deviceStateManager = deviceStateManager; } @PostConstruct @@ -120,10 +140,9 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer boolean sysTenant = TenantId.SYS_TENANT_ID.equals(tenantId) || tenantId == null; if (sysTenant || !request.getStrategy().saveTimeseries() || apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) { KvUtils.validate(request.getEntries(), valueNoXssValidation); - ListenableFuture future = saveTimeseriesInternal(request); + ListenableFuture future = saveTimeseriesInternal(request); if (request.getStrategy().saveTimeseries()) { - FutureCallback callback = getApiUsageCallback(tenantId, request.getCustomerId(), sysTenant, request.getCallback()); - Futures.addCallback(future, callback, tsCallBackExecutor); + Futures.addCallback(future, getApiUsageCallback(tenantId, request.getCustomerId(), sysTenant), tsCallBackExecutor); } } else { request.getCallback().onFailure(new RuntimeException("DB storage writes are disabled due to API limits!")); @@ -131,29 +150,37 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } @Override - public ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request) { + public ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); TimeseriesSaveRequest.Strategy strategy = request.getStrategy(); - ListenableFuture saveFuture; + ListenableFuture resultFuture; + if (strategy.saveTimeseries() && strategy.saveLatest()) { - saveFuture = tsService.save(tenantId, entityId, request.getEntries(), request.getTtl()); + resultFuture = tsService.save(tenantId, entityId, request.getEntries(), request.getTtl()); } else if (strategy.saveLatest()) { - saveFuture = Futures.transform(tsService.saveLatest(tenantId, entityId, request.getEntries()), result -> 0, MoreExecutors.directExecutor()); + resultFuture = tsService.saveLatest(tenantId, entityId, request.getEntries()); } else if (strategy.saveTimeseries()) { - saveFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); + resultFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } else { - saveFuture = Futures.immediateFuture(0); + resultFuture = Futures.immediateFuture(TimeseriesSaveResult.EMPTY); } - addMainCallback(saveFuture, request.getCallback()); + addMainCallback(resultFuture, result -> { + if (strategy.processCalculatedFields()) { + calculatedFieldQueueService.pushRequestToQueue(request, result, request.getCallback()); + } else { + request.getCallback().onSuccess(null); + } + }, t -> request.getCallback().onFailure(t)); + if (strategy.sendWsUpdate()) { - addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); + addWsCallback(resultFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); } if (strategy.saveLatest()) { copyLatestToEntityViews(tenantId, entityId, request.getEntries()); } - return saveFuture; + return resultFuture; } @Override @@ -165,9 +192,68 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void saveAttributesInternal(AttributesSaveRequest request) { log.trace("Executing saveInternal [{}]", request); - ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); - addMainCallback(saveFuture, request.getCallback()); - addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + AttributesSaveRequest.Strategy strategy = request.getStrategy(); + ListenableFuture> resultFuture; + + if (strategy.saveAttributes()) { + resultFuture = attrService.save(tenantId, entityId, request.getScope(), request.getEntries()); + } else { + resultFuture = Futures.immediateFuture(Collections.emptyList()); + } + + addMainCallback(resultFuture, result -> { + if (strategy.processCalculatedFields()) { + calculatedFieldQueueService.pushRequestToQueue(request, result, request.getCallback()); + } else { + request.getCallback().onSuccess(null); + } + }, t -> request.getCallback().onFailure(t)); + + if (shouldSendSharedAttributesUpdatedNotification(request)) { + addMainCallback(resultFuture, success -> clusterService.pushMsgToCore( + DeviceAttributesEventNotificationMsg.onUpdate(tenantId, new DeviceId(entityId.getId()), DataConstants.SHARED_SCOPE, request.getEntries()), null + )); + } + + if (shouldCheckForInactivityTimeoutUpdates(request)) { + findNewInactivityTimeout(request.getEntries()).ifPresent(newInactivityTimeout -> + addMainCallback(resultFuture, success -> deviceStateManager.onDeviceInactivityTimeoutUpdate( + tenantId, new DeviceId(entityId.getId()), newInactivityTimeout, TbCallback.EMPTY) + ) + ); + } + + if (strategy.sendWsUpdate()) { + addWsCallback(resultFuture, success -> onAttributesUpdate(tenantId, entityId, request.getScope().name(), request.getEntries())); + } + } + + private static boolean shouldSendSharedAttributesUpdatedNotification(AttributesSaveRequest request) { + return request.getStrategy().saveAttributes() && shouldSendSharedAttributesNotification(request.getEntityId(), request.getScope(), request.isNotifyDevice()); + } + + private static boolean shouldCheckForInactivityTimeoutUpdates(AttributesSaveRequest request) { + return request.getStrategy().saveAttributes() + && request.getEntityId().getEntityType() == EntityType.DEVICE + && request.getScope() == AttributeScope.SERVER_SCOPE; + } + + private static Optional findNewInactivityTimeout(List entries) { + return entries.stream() + .filter(entry -> Objects.equals(DefaultDeviceStateService.INACTIVITY_TIMEOUT, entry.getKey())) + // Select the entry with the highest version, or if the versions are equal, the one with the most recent update timestamp + .max(comparing(AttributeKvEntry::getVersion, nullsFirst(naturalOrder())).thenComparingLong(AttributeKvEntry::getLastUpdateTs)) + .map(DefaultTelemetrySubscriptionService::parseAsLong); + } + + private static long parseAsLong(KvEntry kve) { + try { + return Long.parseLong(kve.getValueAsString()); + } catch (NumberFormatException e) { + return 0L; + } } @Override @@ -178,9 +264,45 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Override public void deleteAttributesInternal(AttributesDeleteRequest request) { - ListenableFuture> deleteFuture = attrService.removeAll(request.getTenantId(), request.getEntityId(), request.getScope(), request.getKeys()); - addMainCallback(deleteFuture, request.getCallback()); - addWsCallback(deleteFuture, success -> onAttributesDelete(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getKeys(), request.isNotifyDevice())); + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + + ListenableFuture> deleteFuture = attrService.removeAll(tenantId, entityId, request.getScope(), request.getKeys()); + + addMainCallback(deleteFuture, + result -> calculatedFieldQueueService.pushRequestToQueue(request, result, request.getCallback()), + t -> request.getCallback().onFailure(t) + ); + + if (shouldSendSharedAttributesDeletedNotification(request)) { + addMainCallback(deleteFuture, success -> clusterService.pushMsgToCore( + DeviceAttributesEventNotificationMsg.onDelete(tenantId, new DeviceId(entityId.getId()), DataConstants.SHARED_SCOPE, request.getKeys()), null + )); + } + + if (inactivityTimeoutDeleted(request)) { + addMainCallback(deleteFuture, success -> deviceStateManager.onDeviceInactivityTimeoutUpdate( + tenantId, new DeviceId(entityId.getId()), 0L, TbCallback.EMPTY) + ); + } + + addWsCallback(deleteFuture, success -> onAttributesDelete(tenantId, entityId, request.getScope().name(), request.getKeys())); + } + + private static boolean shouldSendSharedAttributesDeletedNotification(AttributesDeleteRequest request) { + return shouldSendSharedAttributesNotification(request.getEntityId(), request.getScope(), request.isNotifyDevice()); + } + + private static boolean shouldSendSharedAttributesNotification(EntityId entityId, AttributeScope scope, boolean notifyDevice) { + return entityId.getEntityType() == EntityType.DEVICE + && scope == AttributeScope.SHARED_SCOPE + && notifyDevice; + } + + private static boolean inactivityTimeoutDeleted(AttributesDeleteRequest request) { + return request.getEntityId().getEntityType() == EntityType.DEVICE + && request.getScope() == AttributeScope.SERVER_SCOPE + && request.getKeys().stream().anyMatch(key -> Objects.equals(DefaultDeviceStateService.INACTIVITY_TIMEOUT, key)); } @Override @@ -199,10 +321,14 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer deleteFuture = tsService.remove(request.getTenantId(), request.getEntityId(), request.getDeleteHistoryQueries()); addWsCallback(deleteFuture, result -> onTimeSeriesDelete(request.getTenantId(), request.getEntityId(), request.getKeys(), result)); } - addMainCallback(deleteFuture, __ -> request.getCallback().onSuccess(request.getKeys()), request.getCallback()::onFailure); + DonAsynchron.withCallback(deleteFuture, result -> { + calculatedFieldQueueService.pushRequestToQueue(request, request.getKeys(), getCalculatedFieldCallback(request.getCallback(), request.getKeys())); + }, safeCallback(getCalculatedFieldCallback(request.getCallback(), request.getKeys())), tsCallBackExecutor); } else { ListenableFuture> deleteFuture = tsService.removeAllLatest(request.getTenantId(), request.getEntityId()); - addMainCallback(deleteFuture, request.getCallback()::onSuccess, request.getCallback()::onFailure); + DonAsynchron.withCallback(deleteFuture, result -> { + calculatedFieldQueueService.pushRequestToQueue(request, request.getKeys(), getCalculatedFieldCallback(request.getCallback(), result)); + }, safeCallback(getCalculatedFieldCallback(request.getCallback(), request.getKeys())), tsCallBackExecutor); } } @@ -228,7 +354,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (entries != null) { Optional tsKvEntry = entries.stream() .filter(entry -> entry.getTs() > startTs && entry.getTs() <= endTs) - .max(Comparator.comparingLong(TsKvEntry::getTs)); + .max(comparingLong(TsKvEntry::getTs)); tsKvEntry.ifPresent(entityViewLatest::add); } } @@ -261,28 +387,22 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } } - private void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice) { - forwardToSubscriptionManagerService(tenantId, entityId, subscriptionManagerService -> { - subscriptionManagerService.onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice, TbCallback.EMPTY); - }, () -> { - return TbSubscriptionUtils.toAttributesUpdateProto(tenantId, entityId, scope, attributes); - }); + private void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes) { + forwardToSubscriptionManagerService(tenantId, entityId, + subscriptionManagerService -> subscriptionManagerService.onAttributesUpdate(tenantId, entityId, scope, attributes, TbCallback.EMPTY), + () -> TbSubscriptionUtils.toAttributesUpdateProto(tenantId, entityId, scope, attributes)); } - private void onAttributesDelete(TenantId tenantId, EntityId entityId, String scope, List keys, boolean notifyDevice) { - forwardToSubscriptionManagerService(tenantId, entityId, subscriptionManagerService -> { - subscriptionManagerService.onAttributesDelete(tenantId, entityId, scope, keys, notifyDevice, TbCallback.EMPTY); - }, () -> { - return TbSubscriptionUtils.toAttributesDeleteProto(tenantId, entityId, scope, keys, notifyDevice); - }); + private void onAttributesDelete(TenantId tenantId, EntityId entityId, String scope, List keys) { + forwardToSubscriptionManagerService(tenantId, entityId, + subscriptionManagerService -> subscriptionManagerService.onAttributesDelete(tenantId, entityId, scope, keys, TbCallback.EMPTY), + () -> TbSubscriptionUtils.toAttributesDeleteProto(tenantId, entityId, scope, keys)); } private void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts) { - forwardToSubscriptionManagerService(tenantId, entityId, subscriptionManagerService -> { - subscriptionManagerService.onTimeSeriesUpdate(tenantId, entityId, ts, TbCallback.EMPTY); - }, () -> { - return TbSubscriptionUtils.toTimeseriesUpdateProto(tenantId, entityId, ts); - }); + forwardToSubscriptionManagerService(tenantId, entityId, + subscriptionManagerService -> subscriptionManagerService.onTimeSeriesUpdate(tenantId, entityId, ts, TbCallback.EMPTY), + () -> TbSubscriptionUtils.toTimeseriesUpdateProto(tenantId, entityId, ts)); } private void onTimeSeriesDelete(TenantId tenantId, EntityId entityId, List keys, List ts) { @@ -302,9 +422,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer subscriptionManagerService.onTimeSeriesUpdate(tenantId, entityId, updated, TbCallback.EMPTY); subscriptionManagerService.onTimeSeriesDelete(tenantId, entityId, deleted, TbCallback.EMPTY); - }, () -> { - return TbSubscriptionUtils.toTimeseriesDeleteProto(tenantId, entityId, keys); - }); + }, () -> TbSubscriptionUtils.toTimeseriesDeleteProto(tenantId, entityId, keys)); } private void addMainCallback(ListenableFuture saveFuture, final FutureCallback callback) { @@ -312,6 +430,10 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer addMainCallback(saveFuture, result -> callback.onSuccess(null), callback::onFailure); } + private void addMainCallback(ListenableFuture saveFuture, Consumer onSuccess) { + addMainCallback(saveFuture, onSuccess, null); + } + private void addMainCallback(ListenableFuture saveFuture, Consumer onSuccess, Consumer onFailure) { DonAsynchron.withCallback(saveFuture, onSuccess, onFailure, tsCallBackExecutor); } @@ -322,19 +444,31 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } } - private FutureCallback getApiUsageCallback(TenantId tenantId, CustomerId customerId, boolean sysTenant, FutureCallback callback) { + private FutureCallback getApiUsageCallback(TenantId tenantId, CustomerId customerId, boolean sysTenant) { return new FutureCallback<>() { @Override - public void onSuccess(Integer result) { - if (!sysTenant && result != null && result > 0) { - apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.STORAGE_DP_COUNT, result); + public void onSuccess(TimeseriesSaveResult result) { + Integer dataPoints = result.getDataPoints(); + if (!sysTenant && dataPoints != null && dataPoints > 0) { + apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.STORAGE_DP_COUNT, dataPoints); } - callback.onSuccess(null); + } + + @Override + public void onFailure(Throwable t) {} + }; + } + + private FutureCallback getCalculatedFieldCallback(FutureCallback> originalCallback, List keys) { + return new FutureCallback<>() { + @Override + public void onSuccess(Void unused) { + originalCallback.onSuccess(keys); } @Override public void onFailure(Throwable t) { - callback.onFailure(t); + originalCallback.onFailure(t); } }; } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java index 380617934d..8a76aa1d14 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java @@ -21,13 +21,14 @@ import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; /** * Created by ashvayka on 27.03.18. */ public interface InternalTelemetryService extends RuleEngineTelemetryService { - ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request); + ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request); void saveAttributesInternal(AttributesSaveRequest request); diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java index 62913a187b..4b60ec923a 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java @@ -93,7 +93,8 @@ public class TbCoreTransportApiService { @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { log.info("Received application ready event. Starting polling for events."); - transportApiTemplate.init(transportApiService); + transportApiTemplate.subscribe(); + transportApiTemplate.launch(transportApiService); } @PreDestroy diff --git a/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java index 90f6b36e0f..7288a8bac9 100644 --- a/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java @@ -229,6 +229,7 @@ public class DefaultWebSocketService implements WebSocketService { } catch (TbRateLimitsException e) { log.debug("{} Failed to handle WS cmd: {}", sessionRef, cmd, e); } catch (Exception e) { + sendError(sessionRef, cmd.getCmdId(), SubscriptionErrorCode.INTERNAL_ERROR, e.getMessage()); log.error("{} Failed to handle WS cmd: {}", sessionRef, cmd, e); } } diff --git a/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java new file mode 100644 index 0000000000..77080c28c8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/CalculatedFieldUtils.java @@ -0,0 +1,152 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils; + +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.util.KvProtoUtil; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsDoubleValProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsRollingArgumentProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; + +import java.util.Optional; +import java.util.TreeMap; +import java.util.UUID; + +public class CalculatedFieldUtils { + + public static CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { + return CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) + .build(); + } + + public static CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { + return CalculatedFieldEntityCtxIdProto.newBuilder() + .setTenantIdMSB(ctxId.tenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(ctxId.tenantId().getId().getLeastSignificantBits()) + .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) + .setEntityType(ctxId.entityId().getEntityType().name()) + .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) + .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) + .build(); + } + + public static CalculatedFieldEntityCtxId fromProto(CalculatedFieldEntityCtxIdProto ctxIdProto) { + TenantId tenantId = TenantId.fromUUID(new UUID(ctxIdProto.getTenantIdMSB(), ctxIdProto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); + return new CalculatedFieldEntityCtxId(tenantId, calculatedFieldId, entityId); + } + + public static CalculatedFieldStateProto toProto(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state) { + CalculatedFieldStateProto.Builder builder = CalculatedFieldStateProto.newBuilder() + .setId(toProto(stateId)) + .setType(state.getType().name()); + + state.getArguments().forEach((argName, argEntry) -> { + if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + builder.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); + } else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) { + builder.addRollingValueArguments(toRollingArgumentProto(argName, rollingArgumentEntry)); + } + }); + return builder.build(); + } + + public static SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { + SingleValueArgumentProto.Builder builder = SingleValueArgumentProto.newBuilder() + .setArgName(argName); + + if (entry.getKvEntryValue() != null) { + builder.setValue(KvProtoUtil.toTsValueProto(entry.getTs(), entry.getKvEntryValue())); + } + + Optional.ofNullable(entry.getVersion()).ifPresent(builder::setVersion); + + return builder.build(); + } + + public static TsRollingArgumentProto toRollingArgumentProto(String argName, TsRollingArgumentEntry entry) { + TsRollingArgumentProto.Builder builder = TsRollingArgumentProto.newBuilder() + .setKey(argName) + .setLimit(entry.getLimit()) + .setTimeWindow(entry.getTimeWindow()); + + entry.getTsRecords().forEach((ts, value) -> builder.addTsValue(TsDoubleValProto.newBuilder().setTs(ts).setValue(value).build())); + + return builder.build(); + } + + public static CalculatedFieldState fromProto(CalculatedFieldStateProto proto) { + if (StringUtils.isEmpty(proto.getType())) { + return null; + } + + CalculatedFieldType type = CalculatedFieldType.valueOf(proto.getType()); + + CalculatedFieldState state = switch (type) { + case SIMPLE -> new SimpleCalculatedFieldState(); + case SCRIPT -> new ScriptCalculatedFieldState(); + }; + + proto.getSingleValueArgumentsList().forEach(argProto -> + state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto))); + + if (CalculatedFieldType.SCRIPT.equals(type)) { + proto.getRollingValueArgumentsList().forEach(argProto -> + state.getArguments().put(argProto.getKey(), fromRollingArgumentProto(argProto))); + } + + return state; + } + + public static SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) { + if (!proto.hasValue()) { + return new SingleValueArgumentEntry(); + } + TsValueProto tsValueProto = proto.getValue(); + return new SingleValueArgumentEntry( + tsValueProto.getTs(), + (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto), + proto.getVersion() + ); + } + + public static TsRollingArgumentEntry fromRollingArgumentProto(TsRollingArgumentProto proto) { + TreeMap tsRecords = new TreeMap<>(); + proto.getTsValueList().forEach(tsValueProto -> tsRecords.put(tsValueProto.getTs(), tsValueProto.getValue())); + return new TsRollingArgumentEntry(tsRecords, proto.getLimit(), proto.getTimeWindow()); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/utils/DebugModeRateLimitsConfig.java b/application/src/main/java/org/thingsboard/server/utils/DebugModeRateLimitsConfig.java new file mode 100644 index 0000000000..99ef186a56 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/DebugModeRateLimitsConfig.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +@Data +public class DebugModeRateLimitsConfig { + + @Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.enabled:true}") + private boolean ruleChainDebugPerTenantLimitsEnabled; + @Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") + private String ruleChainDebugPerTenantLimitsConfiguration; + + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.enabled:true}") + private boolean calculatedFieldDebugPerTenantLimitsEnabled; + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") + private String calculatedFieldDebugPerTenantLimitsConfiguration; + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 7b781c0e3f..2ee3c439c1 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -437,6 +437,8 @@ actors: device_dispatcher_pool_size: "${ACTORS_SYSTEM_DEVICE_DISPATCHER_POOL_SIZE:4}" # Thread pool size for actor system dispatcher that process messages for device actors rule_dispatcher_pool_size: "${ACTORS_SYSTEM_RULE_DISPATCHER_POOL_SIZE:8}" # Thread pool size for actor system dispatcher that process messages for rule engine (chain/node) actors edge_dispatcher_pool_size: "${ACTORS_SYSTEM_EDGE_DISPATCHER_POOL_SIZE:4}" # Thread pool size for actor system dispatcher that process messages for edge actors + cfm_dispatcher_pool_size: "${ACTORS_SYSTEM_CFM_DISPATCHER_POOL_SIZE:2}" # Thread pool size for actor system dispatcher that process messages for CalculatedField manager actors + cfe_dispatcher_pool_size: "${ACTORS_SYSTEM_CFE_DISPATCHER_POOL_SIZE:8}" # Thread pool size for actor system dispatcher that process messages for CalculatedField entity actors tenant: create_components_on_init: "${ACTORS_TENANT_CREATE_COMPONENTS_ON_INIT:true}" # Create components in initialization session: @@ -504,6 +506,14 @@ actors: js_print_interval_ms: "${ACTORS_JS_STATISTICS_PRINT_INTERVAL_MS:10000}" # Actors statistic persistence frequency in milliseconds persist_frequency: "${ACTORS_STATISTICS_PERSIST_FREQUENCY:3600000}" + calculated_fields: + debug_mode_rate_limits_per_tenant: + # Enable/Disable the rate limit of persisted debug events for all calculated fields per tenant + enabled: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_ENABLED:true}" + # The value of DEBUG mode rate limit. By default, no more than 50 thousand events per hour + configuration: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" + # Time in seconds to receive calculation result. + calculation_timeout: "${ACTORS_CALCULATION_TIMEOUT_SEC:5}" debug: settings: @@ -807,7 +817,7 @@ spring: events: # Enable dedicated datasource (a separate database) for events and audit logs. # Before enabling this, make sure you have set up the following tables in the new DB: - # error_event, lc_event, rule_chain_debug_event, rule_node_debug_event, stats_event, audit_log + # error_event, lc_event, rule_chain_debug_event, rule_node_debug_event, stats_event, audit_log, cf_debug_event enabled: "${SPRING_DEDICATED_EVENTS_DATASOURCE_ENABLED:false}" # Database driver for Spring JPA for events datasource driverClassName: "${SPRING_EVENTS_DATASOURCE_DRIVER_CLASS_NAME:org.postgresql.Driver}" @@ -848,6 +858,7 @@ audit-log: "edge": "${AUDIT_LOG_MASK_EDGE:W}" # Edge logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation "tb_resource": "${AUDIT_LOG_MASK_RESOURCE:W}" # TB resource logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation "ota_package": "${AUDIT_LOG_MASK_OTA_PACKAGE:W}" # Ota package logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation + "calculated_field": "${AUDIT_LOG_MASK_CALCULATED_FIELD:W}" # Calculated field logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation sink: # Type of external sink. possible options: none, elasticsearch type: "${AUDIT_LOG_SINK_TYPE:none}" @@ -1589,6 +1600,16 @@ queue: - key: max.poll.records # Amount of records to be returned in a single poll. For Housekeeper reprocessing topic, we should consume messages (tasks) one by one value: "${TB_QUEUE_KAFKA_HOUSEKEEPER_REPROCESSING_MAX_POLL_RECORDS:1}" + edqs.events: + # Key-value properties for Kafka consumer for edqs.events topic + - key: max.poll.records + # Max poll records for edqs.events topic + value: "${TB_QUEUE_KAFKA_EDQS_EVENTS_MAX_POLL_RECORDS:512}" + edqs.state: + # Key-value properties for Kafka consumer for edqs.state topic + - key: max.poll.records + # Max poll records for edqs.state topic + value: "${TB_QUEUE_KAFKA_EDQS_STATE_MAX_POLL_RECORDS:512}" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms @@ -1618,6 +1639,16 @@ queue: edge: "${TB_QUEUE_KAFKA_EDGE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Edge event topic edge-event: "${TB_QUEUE_KAFKA_EDGE_EVENT_TOPIC_PROPERTIES:retention.ms:2592000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Calculated Field topics + calculated-field: "${TB_QUEUE_KAFKA_CF_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Calculated Field State topics + calculated-field-state: "${TB_QUEUE_KAFKA_CF_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:104857600000;partitions:1;min.insync.replicas:1;cleanup.policy:compact}" + # Kafka properties for EDQS events topics. Partitions number must be the same as queue.edqs.partitions + edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1}" + # Kafka properties for EDQS requests topic (default: 3 minutes retention). Partitions number must be the same as queue.edqs.partitions + edqs-requests: "${TB_QUEUE_KAFKA_EDQS_REQUESTS_TOPIC_PROPERTIES:retention.ms:180000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}" + # Kafka properties for EDQS state topic (infinite retention, compaction). Partitions number must be the same as queue.edqs.partitions + edqs-state: "${TB_QUEUE_KAFKA_EDQS_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1;cleanup.policy:compact}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -1647,6 +1678,8 @@ queue: core: # Default topic name topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + # For high-priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_CORE_NOTIFICATIONS_TOPIC:tb_core.notifications}" # Interval in milliseconds to poll messages by Core microservices poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" # Amount of partitions used by Core microservices @@ -1691,6 +1724,43 @@ queue: enabled: "${TB_HOUSEKEEPER_STATS_ENABLED:true}" # Statistics printing interval for Housekeeper print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}" + edqs: + sync: + # Enable/disable EDQS synchronization + enabled: "${TB_EDQS_SYNC_ENABLED:false}" + # Batch size of entities being synced with EDQS + entity_batch_size: "${TB_EDQS_SYNC_ENTITY_BATCH_SIZE:10000}" + # Batch size of timeseries data being synced with EDQS + ts_batch_size: "${TB_EDQS_SYNC_TS_BATCH_SIZE:10000}" + api: + # Whether to forward entity data query requests to EDQS (otherwise use PostgreSQL implementation) + supported: "${TB_EDQS_API_SUPPORTED:false}" + # Whether to auto-enable EDQS API (if queue.edqs.api.supported is true) when sync of data to Kafka is finished + auto_enable: "${TB_EDQS_API_AUTO_ENABLE:true}" + # Mode of EDQS: local (for monolith) or remote (with separate EDQS microservices) + mode: "${TB_EDQS_MODE:local}" + local: + # Path to RocksDB for EDQS backup when running in local mode + rocksdb_path: "${TB_EDQS_ROCKSDB_PATH:${user.home}/.rocksdb/edqs}" + # Number of partitions for EDQS topics + partitions: "${TB_EDQS_PARTITIONS:12}" + # EDQS partitioning strategy: tenant (partition is resolved by tenant id) or none (no specific strategy, resolving by message key) + partitioning_strategy: "${TB_EDQS_PARTITIONING_STRATEGY:tenant}" + # EDQS requests topic + requests_topic: "${TB_EDQS_REQUESTS_TOPIC:edqs.requests}" + # EDQS responses topic + responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}" + # Poll interval for EDQS topics + poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}" + # Maximum amount of pending requests to EDQS + max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}" + # Maximum timeout for requests to EDQS + max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:20000}" + stats: + # Enable/disable statistics for EDQS + enabled: "${TB_EDQS_STATS_ENABLED:true}" + # Statistics printing interval for EDQS + print-interval-ms: "${TB_EDQS_STATS_PRINT_INTERVAL_MS:300000}" vc: # Default topic name @@ -1723,6 +1793,8 @@ queue: rule-engine: # Deprecated. It will be removed in the nearest releases topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" + # For high-priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_RULE_ENGINE_NOTIFICATIONS_TOPIC:tb_rule_engine.notifications}" # Interval in milliseconds to poll messages by Rule Engine poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" # Timeout for processing a message pack of Rule Engine @@ -1738,6 +1810,23 @@ queue: topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:15}" # Size of the thread pool that handles such operations as partition changes, config updates, queue deletion management-thread-pool-size: "${TB_QUEUE_RULE_ENGINE_MGMT_THREAD_POOL_SIZE:12}" + calculated_fields: + # Topic name for Calculated Field (CF) events from Rule Engine + event_topic: "${TB_QUEUE_CF_EVENT_TOPIC:tb_cf_event}" + # Topic name for Calculated Field (CF) compacted states + state_topic: "${TB_QUEUE_CF_STATE_TOPIC:tb_cf_state}" + # For high-priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_CF_NOTIFICATIONS_TOPIC:calculated_field.notifications}" + # Interval in milliseconds to poll messages by CF (Rule Engine) microservices + poll_interval: "${TB_QUEUE_CF_POLL_INTERVAL_MS:25}" + # Amount of partitions used by CF microservices + partitions: "${TB_QUEUE_CF_PARTITIONS:10}" + # Timeout for processing a message pack by CF microservices + pack_processing_timeout: "${TB_QUEUE_CF_PACK_PROCESSING_TIMEOUT_MS:60000}" + # Thread pool size for processing of the incoming messages + pool_size: "${TB_QUEUE_CF_POOL_SIZE:8}" + # RocksDB path for storing CF states + rocks_db_path: "${TB_QUEUE_CF_ROCKS_DB_PATH:${user.home}/.rocksdb/cf_states}" transport: # For high-priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" @@ -1746,6 +1835,10 @@ queue: edge: # Default topic name topic: "${TB_QUEUE_EDGE_TOPIC:tb_edge}" + # For high-priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_EDGE_NOTIFICATIONS_TOPIC:tb_edge.notifications}" + # For edge events messages + event_notifications_topic: "${TB_QUEUE_EDGE_EVENT_NOTIFICATIONS_TOPIC:tb_edge_event.notifications}" # Amount of partitions used by Edge services partitions: "${TB_QUEUE_EDGE_PARTITIONS:10}" # Poll interval for topics related to Edge services diff --git a/application/src/test/java/org/thingsboard/server/actors/rule/DefaultTbContextTest.java b/application/src/test/java/org/thingsboard/server/actors/rule/DefaultTbContextTest.java index ea900afd4e..54205327ca 100644 --- a/application/src/test/java/org/thingsboard/server/actors/rule/DefaultTbContextTest.java +++ b/application/src/test/java/org/thingsboard/server/actors/rule/DefaultTbContextTest.java @@ -131,11 +131,87 @@ class DefaultTbContextTest { defaultTbContext.input(msg, ruleChainId); // THEN + then(clusterService).should().pushMsgToRuleEngine(eq(tpi), eq(msg.getId()), any(), any()); + } + + @MethodSource + @ParameterizedTest + public void givenMsgWithQueueName_whenEnqueue_thenVerifyEnqueueWithCorrectTpi(String queueName) { + // GIVEN + var tpi = resolve(queueName); + + given(mainCtxMock.resolve(eq(ServiceType.TB_RULE_ENGINE), eq(queueName), eq(TENANT_ID), eq(TENANT_ID))).willReturn(tpi); + var clusterService = mock(TbClusterService.class); + given(mainCtxMock.getClusterService()).willReturn(clusterService); + var callbackMock = mock(TbMsgCallback.class); + given(callbackMock.isMsgValid()).willReturn(true); + var ruleNode = new RuleNode(RULE_NODE_ID); + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(TENANT_ID) + .queueName(queueName) + .copyMetaData(TbMsgMetaData.EMPTY) + .data(TbMsg.EMPTY_STRING) + .callback(callbackMock) + .build(); + + ruleNode.setRuleChainId(RULE_CHAIN_ID); + ruleNode.setDebugSettings(DebugSettings.failures()); + given(nodeCtxMock.getTenantId()).willReturn(TENANT_ID); + + // WHEN + defaultTbContext.enqueue(msg, () -> {}, t -> {}); + + // THEN + then(clusterService).should().pushMsgToRuleEngine(eq(tpi), eq(msg.getId()), any(), any()); + } + @MethodSource + @ParameterizedTest + public void givenMsgAndQueueName_whenEnqueue_thenVerifyEnqueueWithCorrectTpi(String queueName) { + // GIVEN + var tpi = resolve(queueName); + + given(mainCtxMock.resolve(eq(ServiceType.TB_RULE_ENGINE), eq(queueName), eq(TENANT_ID), eq(TENANT_ID))).willReturn(tpi); + var clusterService = mock(TbClusterService.class); + given(mainCtxMock.getClusterService()).willReturn(clusterService); + var callbackMock = mock(TbMsgCallback.class); + given(callbackMock.isMsgValid()).willReturn(true); + var ruleNode = new RuleNode(RULE_NODE_ID); + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(TENANT_ID) + .copyMetaData(TbMsgMetaData.EMPTY) + .data(TbMsg.EMPTY_STRING) + .callback(callbackMock) + .build(); + + ruleNode.setRuleChainId(RULE_CHAIN_ID); + ruleNode.setDebugSettings(DebugSettings.failures()); + given(nodeCtxMock.getTenantId()).willReturn(TENANT_ID); + + // WHEN + defaultTbContext.enqueue(msg, queueName, () -> {}, t -> {}); + + // THEN then(clusterService).should().pushMsgToRuleEngine(eq(tpi), eq(msg.getId()), any(), any()); } private static Stream givenMsgWithQueueName_whenInput_thenVerifyEnqueueWithCorrectTpi() { + return testQueueNames(); + } + + private static Stream givenMsgWithQueueName_whenEnqueue_thenVerifyEnqueueWithCorrectTpi() { + return testQueueNames(); + } + + private static Stream givenMsgAndQueueName_whenEnqueue_thenVerifyEnqueueWithCorrectTpi() { + return testQueueNames(); + } + + private static Stream testQueueNames() { return Stream.of("Main", "Test", null); } diff --git a/application/src/test/java/org/thingsboard/server/actors/tenant/TenantActorTest.java b/application/src/test/java/org/thingsboard/server/actors/tenant/TenantActorTest.java index 50349907fa..d8fac2b936 100644 --- a/application/src/test/java/org/thingsboard/server/actors/tenant/TenantActorTest.java +++ b/application/src/test/java/org/thingsboard/server/actors/tenant/TenantActorTest.java @@ -27,6 +27,7 @@ import org.thingsboard.server.actors.TbActorSystemSettings; import org.thingsboard.server.actors.TbEntityActorId; import org.thingsboard.server.actors.ruleChain.RuleChainActor; import org.thingsboard.server.actors.ruleChain.RuleChainToRuleChainMsg; +import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.RuleChainErrorActor; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.id.DeviceId; @@ -116,6 +117,7 @@ public class TenantActorTest { TbActorSystemSettings settings = new TbActorSystemSettings(0, 0, 0); TbActorSystem system = spy(new DefaultTbActorSystem(settings)); system.createDispatcher(RULE_DISPATCHER_NAME, mock()); + system.createDispatcher(DefaultActorService.CF_MANAGER_DISPATCHER_NAME, mock()); TbActorMailbox tenantCtx = new TbActorMailbox(system, settings, null, mock(), mock(), null); tenantActor.init(tenantCtx); diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java new file mode 100644 index 0000000000..9ddf0f0a04 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -0,0 +1,459 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cf; + +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.controller.CalculatedFieldControllerTest; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +@DaoSqlTest +public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTest { + + @BeforeEach + void setUp() throws Exception { + loginTenantAdmin(); + } + + @Test + public void testSimpleCalculatedFieldWhenAllTelemetryPresent() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":25}")); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"deviceTemperature\":40}")); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("C to F"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + argument.setDefaultValue("12"); // not used because real telemetry value in db is present + config.setArguments(Map.of("T", argument)); + config.setExpression("(T * 9/5) + 32"); + + Output output = new Output(); + output.setName("fahrenheitTemp"); + output.setType(OutputType.TIME_SERIES); + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + await().alias("create CF -> perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("77.0"); + }); + + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); + + await().alias("update telemetry -> recalculate state").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + }); + + Output savedOutput = savedCalculatedField.getConfiguration().getOutput(); + savedOutput.setType(OutputType.ATTRIBUTES); + savedOutput.setScope(AttributeScope.SERVER_SCOPE); + savedOutput.setName("temperatureF"); + savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); + + await().alias("update CF output -> perform calculation with updated output").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("86.0"); + }); + + Argument savedArgument = savedCalculatedField.getConfiguration().getArguments().get("T"); + savedArgument.setRefEntityKey(new ReferencedEntityKey("deviceTemperature", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); + savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); + + await().alias("update CF argument -> perform calculation with new argument").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); + }); + + savedCalculatedField.getConfiguration().setExpression("1.8 * T + 32"); + savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); + + await().alias("update CF expression -> perform calculation with new expression").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); + }); + } + + @Test + public void testSimpleCalculatedFieldWhenNotAllTelemetryPresent() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("C to F"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + config.setArguments(Map.of("T", argument)); + config.setExpression("(T * 9/5) + 32"); + + Output output = new Output(); + output.setName("fahrenheitTemp"); + output.setType(OutputType.TIME_SERIES); + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + await().alias("create CF -> state is not ready -> no calculation performed").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + }); + + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); + + await().alias("update telemetry -> perform calculation").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + }); + } + + @Test + public void testSimpleCalculatedFieldWhenNotAllTelemetryPresentButDefaultValueIsSet() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("C to F"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + argument.setDefaultValue("12"); + config.setArguments(Map.of("T", argument)); + config.setExpression("(T * 9/5) + 32"); + + Output output = new Output(); + output.setName("fahrenheitTemp"); + output.setType(OutputType.TIME_SERIES); + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + await().alias("create CF -> perform initial calculation with default value").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("53.6"); + }); + + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); + + await().alias("update telemetry -> recalculate state").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + }); + } + + @Test + public void testSimpleCalculatedFieldWhenEntityIdIsProfile() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":40}")); + + AssetProfile assetProfile = doPost("/api/assetProfile", createAssetProfile("Test Asset Profile"), AssetProfile.class); + + Asset asset1 = createAsset("Test asset 1", assetProfile.getId()); + doPost("/api/plugins/telemetry/ASSET/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":11}")); + + Asset asset2 = createAsset("Test asset 2", assetProfile.getId()); + doPost("/api/plugins/telemetry/ASSET/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":12}")); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(assetProfile.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("z = x + y"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument1 = new Argument(); + ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("y", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE); + argument1.setRefEntityKey(refEntityKey1); + + Argument argument2 = new Argument(); + argument2.setRefEntityId(testDevice.getId()); + ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("x", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE); + argument2.setRefEntityKey(refEntityKey2); + + config.setArguments(Map.of("x", argument2, "y", argument1)); + + config.setExpression("x + y"); + + Output output = new Output(); + output.setName("z"); + output.setType(OutputType.ATTRIBUTES); + output.setScope(AttributeScope.SERVER_SCOPE); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + await().alias("create CF and perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("51.0"); + + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("52.0"); + }); + + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":25}")); + + await().alias("update device telemetry -> recalculate state for all assets").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("36.0"); + + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + }); + + doPost("/api/plugins/telemetry/ASSET/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":15}")); + + await().alias("update asset 1 telemetry -> recalculate state only for asset 1").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); + + // result of asset 2 (no changes) + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + }); + + doPost("/api/plugins/telemetry/ASSET/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":5}")); + + await().alias("update asset 2 telemetry -> recalculate state only for asset 2").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 (no changes) + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); + + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("30.0"); + }); + + Asset asset3 = createAsset("Test asset 3", assetProfile.getId()); + doPost("/api/plugins/telemetry/ASSET/" + asset3.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":13}")); + + Asset finalAsset3 = asset3; + await().alias("add new entity to profile -> calculate state for new entity").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 3 + ArrayNode z3 = getServerAttributes(finalAsset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("38.0"); + }); + + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":20}")); + + await().alias("update device telemetry -> recalculate state for all assets").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("35.0"); + + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("25.0"); + + // result of asset 3 + ArrayNode z3 = getServerAttributes(finalAsset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + }); + + // update profile for asset 3 -> delete state for asset 3 + AssetProfile newAssetProfile = doPost("/api/assetProfile", createAssetProfile("New Asset Profile"), AssetProfile.class); + asset3.setAssetProfileId(newAssetProfile.getId()); + asset3 = doPost("/api/asset", asset3, Asset.class); + + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":15}")); + + Asset updatedAsset3 = asset3; + await().alias("update device telemetry -> recalculate state for asset 1 and asset 2").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("30.0"); + + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("20.0"); + + // no changes for asset 3 + ArrayNode z3 = getServerAttributes(updatedAsset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + }); + } + + @Test + public void testSimpleCalculatedFieldWhenExpressionIsInvalid() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":25}")); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("C to F"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + argument.setDefaultValue("12"); // not used because real telemetry value in db is present + config.setArguments(Map.of("T", argument)); + config.setExpression("(T * 9/0) + 32"); + + Output output = new Output(); + output.setName("fahrenheitTemp"); + output.setType(OutputType.TIME_SERIES); + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + await().alias("create CF -> ctx is not initialized -> no calculation perform").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + }); + + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); + + await().alias("update telemetry -> ctx is not initialized -> no calculation perform").atMost(TIMEOUT, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + }); + } + + private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { + return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class); + } + + private ArrayNode getServerAttributes(EntityId entityId, String... keys) throws Exception { + return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/attributes/SERVER_SCOPE?keys=" + String.join(",", keys), ArrayNode.class); + } + + private Asset createAsset(String name, AssetProfileId assetProfileId) { + Asset asset = new Asset(); + asset.setName(name); + asset.setAssetProfileId(assetProfileId); + return doPost("/api/asset", asset, Asset.class); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 975c3b426b..e90aecb41e 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -402,7 +402,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { } catch (Exception e) { throw new RuntimeException(e); } - Awaitility.await("all tasks processed").atMost(60, TimeUnit.SECONDS).during(300, TimeUnit.MILLISECONDS) + Awaitility.await("all tasks processed").atMost(90, TimeUnit.SECONDS).during(300, TimeUnit.MILLISECONDS) .until(() -> storage.getLag("tb_housekeeper") == 0); } diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java new file mode 100644 index 0000000000..ee66f664cc --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -0,0 +1,163 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@DaoSqlTest +public class CalculatedFieldControllerTest extends AbstractControllerTest { + + private Tenant savedTenant; + + @Before + public void beforeTest() throws Exception { + loginSysAdmin(); + + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + savedTenant = saveTenant(tenant); + assertThat(savedTenant).isNotNull(); + + User tenantAdmin = new User(); + tenantAdmin.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin.setTenantId(savedTenant.getId()); + tenantAdmin.setEmail("tenant2@thingsboard.org"); + tenantAdmin.setFirstName("Joe"); + tenantAdmin.setLastName("Downs"); + + createUserAndLogin(tenantAdmin, "testPassword1"); + } + + @After + public void afterTest() throws Exception { + loginSysAdmin(); + + deleteTenant(savedTenant.getId()); + } + + @Test + public void testSaveCalculatedField() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + CalculatedField calculatedField = getCalculatedField(testDevice.getId()); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + assertThat(savedCalculatedField).isNotNull(); + assertThat(savedCalculatedField.getId()).isNotNull(); + assertThat(savedCalculatedField.getCreatedTime()).isGreaterThan(0); + assertThat(savedCalculatedField.getTenantId()).isEqualTo(savedTenant.getId()); + assertThat(savedCalculatedField.getEntityId()).isEqualTo(calculatedField.getEntityId()); + assertThat(savedCalculatedField.getType()).isEqualTo(calculatedField.getType()); + assertThat(savedCalculatedField.getName()).isEqualTo(calculatedField.getName()); + assertThat(savedCalculatedField.getConfiguration()).isEqualTo(getCalculatedFieldConfig(testDevice.getId())); + assertThat(savedCalculatedField.getVersion()).isEqualTo(1L); + + savedCalculatedField.setName("Test CF"); + + CalculatedField updatedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); + + assertThat(updatedCalculatedField.getName()).isEqualTo(savedCalculatedField.getName()); + assertThat(updatedCalculatedField.getVersion()).isEqualTo(savedCalculatedField.getVersion() + 1); + + doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) + .andExpect(status().isOk()); + } + + @Test + public void testGetCalculatedFieldById() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + CalculatedField calculatedField = getCalculatedField(testDevice.getId()); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + CalculatedField fetchedCalculatedField = doGet("/api/calculatedField/" + savedCalculatedField.getId().getId(), CalculatedField.class); + + assertThat(fetchedCalculatedField).isNotNull(); + assertThat(fetchedCalculatedField).isEqualTo(savedCalculatedField); + + doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) + .andExpect(status().isOk()); + } + + @Test + public void testDeleteCalculatedField() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + CalculatedField calculatedField = getCalculatedField(testDevice.getId()); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + assertThat(savedCalculatedField).isNotNull(); + + doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) + .andExpect(status().isOk()); + doGet("/api/calculatedField/" + savedCalculatedField.getId().getId()).andExpect(status().isNotFound()); + } + + private CalculatedField getCalculatedField(DeviceId deviceId) { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(deviceId); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(getCalculatedFieldConfig(null)); + calculatedField.setVersion(1L); + return calculatedField; + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + argument.setRefEntityId(referencedEntityId); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + + config.setArguments(Map.of("T", argument)); + + config.setExpression("T - (100 - H) / 5"); + + Output output = new Output(); + output.setName("output"); + output.setType(OutputType.TIME_SERIES); + + config.setOutput(output); + + return config; + } + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java index 9688ce8758..320e3815f6 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java @@ -27,6 +27,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Primary; import org.springframework.test.context.ContextConfiguration; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.Device; @@ -60,6 +61,7 @@ import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; +import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.containsString; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.common.data.DataConstants.DEFAULT_DEVICE_TYPE; @@ -79,6 +81,8 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { @Autowired private DeviceProfileDao deviceProfileDao; + static final String LWM2M_PROFILE_JSON = "{\"name\":\"lwm2m profile\",\"type\":\"DEFAULT\",\"image\":null,\"defaultQueueName\":null,\"transportType\":\"LWM2M\",\"provisionType\":\"DISABLED\",\"description\":\"\",\"profileData\":{\"configuration\":{\"type\":\"DEFAULT\"},\"transportConfiguration\":{\"observeAttr\":{\"observe\":[],\"attribute\":[],\"telemetry\":[\"/11_1.1/0/0\"],\"keyName\":{\"/11_1.1/0/0\":\"profileName\"},\"attributeLwm2m\":{}},\"bootstrap\":[{\"shortServerId\":123,\"bootstrapServerIs\":false,\"host\":\"0.0.0.0\",\"port\":5685,\"clientHoldOffTime\":1,\"serverPublicKey\":\"\",\"serverCertificate\":\"\",\"bootstrapServerAccountTimeout\":0,\"lifetime\":300,\"defaultMinPeriod\":1,\"notifIfDisabled\":true,\"binding\":\"U\",\"securityMode\":\"NO_SEC\"}],\"clientLwM2mSettings\":{\"clientOnlyObserveAfterConnect\":1,\"fwUpdateStrategy\":1,\"swUpdateStrategy\":1,\"powerMode\":\"DRX\",\"edrxCycle\":81000,\"psmActivityTimer\":10000,\"pagingTransmissionWindow\":10000,\"defaultObjectIDVer\":\"1.0\"},\"bootstrapServerUpdateEnable\":false,\"type\":\"LWM2M\"},\"alarms\":null,\"provisionConfiguration\":{\"type\":\"DISABLED\"}}}"; + static class Config { @Bean @Primary @@ -119,7 +123,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { Mockito.reset(tbClusterService, auditLogService); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile savedDeviceProfile = saveDeviceProfile(deviceProfile); Assert.assertNotNull(savedDeviceProfile); Assert.assertNotNull(savedDeviceProfile.getId()); Assert.assertTrue(savedDeviceProfile.getCreatedTime() > 0); @@ -135,7 +139,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { ActionType.ADDED); savedDeviceProfile.setName("New device profile"); - doPost("/api/deviceProfile", savedDeviceProfile, DeviceProfile.class); + saveDeviceProfile(savedDeviceProfile); DeviceProfile foundDeviceProfile = doGet("/api/deviceProfile/" + savedDeviceProfile.getId().getId().toString(), DeviceProfile.class); Assert.assertEquals(savedDeviceProfile.getName(), foundDeviceProfile.getName()); @@ -162,7 +166,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { @Test public void testFindDeviceProfileById() throws Exception { DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile savedDeviceProfile = saveDeviceProfile(deviceProfile); DeviceProfile foundDeviceProfile = doGet("/api/deviceProfile/" + savedDeviceProfile.getId().getId().toString(), DeviceProfile.class); Assert.assertNotNull(foundDeviceProfile); Assert.assertEquals(savedDeviceProfile, foundDeviceProfile); @@ -171,7 +175,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { @Test public void whenGetDeviceProfileById_thenPermissionsAreChecked() throws Exception { DeviceProfile deviceProfile = createDeviceProfile("Device profile 1", null); - deviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + deviceProfile = saveDeviceProfile(deviceProfile); loginDifferentTenant(); @@ -183,7 +187,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { @Test public void testFindDeviceProfileInfoById() throws Exception { DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile savedDeviceProfile = saveDeviceProfile(deviceProfile); DeviceProfileInfo foundDeviceProfileInfo = doGet("/api/deviceProfileInfo/" + savedDeviceProfile.getId().getId().toString(), DeviceProfileInfo.class); Assert.assertNotNull(foundDeviceProfileInfo); Assert.assertEquals(savedDeviceProfile.getId(), foundDeviceProfileInfo.getId()); @@ -213,7 +217,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { @Test public void whenGetDeviceProfileInfoById_thenPermissionsAreChecked() throws Exception { DeviceProfile deviceProfile = createDeviceProfile("Device profile 1", null); - deviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + deviceProfile = saveDeviceProfile(deviceProfile); loginDifferentTenant(); doGet("/api/deviceProfileInfo/" + deviceProfile.getId()) @@ -235,7 +239,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { @Test public void testSetDefaultDeviceProfile() throws Exception { DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile 1"); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile savedDeviceProfile = saveDeviceProfile(deviceProfile); Mockito.reset(tbClusterService, auditLogService); @@ -328,7 +332,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { @Test public void testChangeDeviceProfileTypeNull() throws Exception { DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile savedDeviceProfile = saveDeviceProfile(deviceProfile); Mockito.reset(tbClusterService, auditLogService); @@ -345,7 +349,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { @Test public void testChangeDeviceProfileTransportTypeWithExistingDevices() throws Exception { DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile savedDeviceProfile = saveDeviceProfile(deviceProfile); Device device = new Device(); device.setName("Test device"); device.setType("default"); @@ -367,7 +371,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { @Test public void testDeleteDeviceProfileWithExistingDevice() throws Exception { DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile savedDeviceProfile = saveDeviceProfile(deviceProfile); Device device = new Device(); device.setName("Test device"); @@ -419,7 +423,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { public void testSaveDeviceProfileWithFirmwareFromDifferentTenant() throws Exception { loginDifferentTenant(); DeviceProfile differentProfile = createDeviceProfile("Different profile"); - differentProfile = doPost("/api/deviceProfile", differentProfile, DeviceProfile.class); + differentProfile = saveDeviceProfile(differentProfile); SaveOtaPackageInfoRequest firmwareInfo = new SaveOtaPackageInfoRequest(); firmwareInfo.setDeviceProfileId(differentProfile.getId()); firmwareInfo.setType(FIRMWARE); @@ -441,7 +445,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { public void testSaveDeviceProfileWithSoftwareFromDifferentTenant() throws Exception { loginDifferentTenant(); DeviceProfile differentProfile = createDeviceProfile("Different profile"); - differentProfile = doPost("/api/deviceProfile", differentProfile, DeviceProfile.class); + differentProfile = saveDeviceProfile(differentProfile); SaveOtaPackageInfoRequest softwareInfo = new SaveOtaPackageInfoRequest(); softwareInfo.setDeviceProfileId(differentProfile.getId()); softwareInfo.setType(SOFTWARE); @@ -462,7 +466,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { @Test public void testDeleteDeviceProfile() throws Exception { DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile savedDeviceProfile = saveDeviceProfile(deviceProfile); Mockito.reset(tbClusterService, auditLogService); @@ -495,7 +499,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { int cntEntity = 28; for (int i = 0; i < cntEntity; i++) { DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile" + i); - deviceProfiles.add(doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class)); + deviceProfiles.add(saveDeviceProfile(deviceProfile)); } testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(new DeviceProfile(), new DeviceProfile(), @@ -552,7 +556,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { for (int i = 0; i < 28; i++) { DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile" + i); - deviceProfiles.add(doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class)); + deviceProfiles.add(saveDeviceProfile(deviceProfile)); } List loadedDeviceProfileInfos = new ArrayList<>(); @@ -961,7 +965,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { JsonTransportPayloadConfiguration jsonTransportPayloadConfiguration = new JsonTransportPayloadConfiguration(); MqttDeviceProfileTransportConfiguration mqttDeviceProfileTransportConfiguration = this.createMqttDeviceProfileTransportConfiguration(jsonTransportPayloadConfiguration, true); DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile", mqttDeviceProfileTransportConfiguration); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile savedDeviceProfile = saveDeviceProfile(deviceProfile); Assert.assertNotNull(savedDeviceProfile); Assert.assertEquals(savedDeviceProfile.getTransportType(), DeviceTransportType.MQTT); Assert.assertTrue(savedDeviceProfile.getProfileData().getTransportConfiguration() instanceof MqttDeviceProfileTransportConfiguration); @@ -979,7 +983,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { "v1/devices/me/telemetry", "v1/devices/me/attributes", "v1/devices/me/subscribeattributes"); DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile", mqttDeviceProfileTransportConfiguration); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile savedDeviceProfile = saveDeviceProfile(deviceProfile); Assert.assertNotNull(savedDeviceProfile); Assert.assertEquals(savedDeviceProfile.getTransportType(), DeviceTransportType.MQTT); Assert.assertTrue(savedDeviceProfile.getProfileData().getTransportConfiguration() instanceof MqttDeviceProfileTransportConfiguration); @@ -997,7 +1001,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = this.createProtoTransportPayloadConfiguration(schema, schema, null, null); MqttDeviceProfileTransportConfiguration mqttDeviceProfileTransportConfiguration = this.createMqttDeviceProfileTransportConfiguration(protoTransportPayloadConfiguration, false); DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile", mqttDeviceProfileTransportConfiguration); - DeviceProfile savedDeviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + DeviceProfile savedDeviceProfile = saveDeviceProfile(deviceProfile); Assert.assertNotNull(savedDeviceProfile); DeviceProfile foundDeviceProfile = doGet("/api/deviceProfile/" + savedDeviceProfile.getId().getId().toString(), DeviceProfile.class); Assert.assertEquals(savedDeviceProfile, foundDeviceProfile); @@ -1036,14 +1040,14 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { @Test public void testDeleteDeviceProfileWithDeleteRelationsOk() throws Exception { - DeviceProfileId deviceProfileId = savedDeviceProfile("DeviceProfile for Test WithRelationsOk").getId(); + DeviceProfileId deviceProfileId = saveDeviceProfile("DeviceProfile for Test WithRelationsOk").getId(); testEntityDaoWithRelationsOk(savedTenant.getId(), deviceProfileId, "/api/deviceProfile/" + deviceProfileId); } @Ignore @Test public void testDeleteDeviceProfileExceptionWithRelationsTransactional() throws Exception { - DeviceProfileId deviceProfileId = savedDeviceProfile("DeviceProfile for Test WithRelations Transactional Exception").getId(); + DeviceProfileId deviceProfileId = saveDeviceProfile("DeviceProfile for Test WithRelations Transactional Exception").getId(); testEntityDaoWithRelationsTransactionalException(deviceProfileDao, savedTenant.getId(), deviceProfileId, "/api/deviceProfile/" + deviceProfileId); } @@ -1103,8 +1107,36 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { Assert.assertEquals(count, deviceProfileNames.size()); } - private DeviceProfile savedDeviceProfile(String name) { + @Test + public void testSaveDeviceProfileWithOutdatedVersion() throws Exception { + DeviceProfile deviceProfile = JacksonUtil.fromString(LWM2M_PROFILE_JSON, DeviceProfile.class); + deviceProfile.setName("Device profile v1.0"); + deviceProfile = saveDeviceProfile(deviceProfile); + assertThat(deviceProfile.getVersion()).isOne(); + + deviceProfile.setName("Device profile v2.0"); + deviceProfile = saveDeviceProfile(deviceProfile); + assertThat(deviceProfile.getVersion()).isEqualTo(2); + + deviceProfile.setName("Device profile v1.1"); + deviceProfile.setVersion(1L); + String response = doPost("/api/deviceProfile", deviceProfile).andExpect(status().isConflict()) + .andReturn().getResponse().getContentAsString(); + assertThat(JacksonUtil.toJsonNode(response).get("message").asText()) + .containsIgnoringCase("already changed by someone else"); + + deviceProfile.setVersion(null); // overriding entity + deviceProfile = saveDeviceProfile(deviceProfile); + assertThat(deviceProfile.getName()).isEqualTo("Device profile v1.1"); + assertThat(deviceProfile.getVersion()).isEqualTo(3); + } + + private DeviceProfile saveDeviceProfile(String name) { DeviceProfile deviceProfile = createDeviceProfile(name); + return saveDeviceProfile(deviceProfile); + } + + private DeviceProfile saveDeviceProfile(DeviceProfile deviceProfile) { return doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); } diff --git a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java new file mode 100644 index 0000000000..ce5221ab89 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java @@ -0,0 +1,72 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import org.junit.Before; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.TestPropertySource; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.msg.edqs.EdqsApiService; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.edqs.state.EdqsStateService; +import org.thingsboard.server.edqs.util.EdqsRocksDb; + +import java.util.concurrent.TimeUnit; + +import static org.awaitility.Awaitility.await; + +@DaoSqlTest +@TestPropertySource(properties = { +// "queue.type=kafka", // uncomment to use Kafka +// "queue.kafka.bootstrap.servers=10.7.1.254:9092", + "queue.edqs.sync.enabled=true", + "queue.edqs.api.supported=true", + "queue.edqs.api.auto_enable=true", + "queue.edqs.mode=local" +}) +public class EdqsEntityQueryControllerTest extends EntityQueryControllerTest { + + @Autowired + private EdqsApiService edqsApiService; + + @Autowired + private EdqsStateService edqsStateService; + + @MockBean // so that we don't do backup for tests + private EdqsRocksDb edqsRocksDb; + + @Before + public void before() { + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsApiService.isEnabled() && edqsStateService.isReady()); + } + + @Override + protected PageData findByQueryAndCheck(EntityDataQuery query, int expectedResultSize) { + return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findByQuery(query), + result -> result.getTotalElements() == expectedResultSize); + } + + @Override + protected Long countByQueryAndCheck(EntityCountQuery query, long expectedResult) { + return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> countByQuery(query), + result -> result == expectedResult); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java index ac618d7f96..011399e883 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java @@ -18,12 +18,14 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import org.awaitility.Awaitility; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.web.servlet.ResultActions; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; @@ -49,6 +51,7 @@ import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; import org.thingsboard.server.common.data.query.EntityListFilter; import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.common.data.query.FilterPredicateValue; @@ -70,9 +73,11 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.function.BiPredicate; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @DaoSqlTest @@ -130,36 +135,25 @@ public class EntityQueryControllerTest extends AbstractControllerTest { filter.setDeviceNameFilter(""); EntityCountQuery countQuery = new EntityCountQuery(filter); - - Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(97, count.longValue()); + countByQueryAndCheck(countQuery, 97); filter.setDeviceTypes(List.of("unknown")); - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(0, count.longValue()); + countByQueryAndCheck(countQuery, 0); filter.setDeviceTypes(List.of("default")); filter.setDeviceNameFilter("Device1"); - - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(11, count.longValue()); + countByQueryAndCheck(countQuery, 11); EntityListFilter entityListFilter = new EntityListFilter(); entityListFilter.setEntityType(EntityType.DEVICE); entityListFilter.setEntityList(devices.stream().map(Device::getId).map(DeviceId::toString).collect(Collectors.toList())); - countQuery = new EntityCountQuery(entityListFilter); - - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(97, count.longValue()); + countByQueryAndCheck(countQuery, 97); EntityTypeFilter filter2 = new EntityTypeFilter(); filter2.setEntityType(EntityType.DEVICE); - - EntityCountQuery countQuery2 = new EntityCountQuery(filter2); - - Long count2 = doPostWithResponse("/api/entitiesQuery/count", countQuery2, Long.class); - Assert.assertEquals(97, count2.longValue()); + countQuery = new EntityCountQuery(filter2); + countByQueryAndCheck(countQuery, 97); } @Test @@ -169,51 +163,44 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityTypeFilter allDeviceFilter = new EntityTypeFilter(); allDeviceFilter.setEntityType(EntityType.DEVICE); EntityCountQuery query = new EntityCountQuery(allDeviceFilter); - Long initialCount = doPostWithResponse("/api/entitiesQuery/count", query, Long.class); + countByQueryAndCheck(query, 0); loginTenantAdmin(); List devices = new ArrayList<>(); + String devicePrefix = "Device" + RandomStringUtils.randomAlphabetic(5); for (int i = 0; i < 97; i++) { Device device = new Device(); - device.setName("Device" + i); + device.setName(devicePrefix + i); device.setType("default"); device.setLabel("testLabel" + (int) (Math.random() * 1000)); devices.add(doPost("/api/device", device, Device.class)); Thread.sleep(1); } DeviceTypeFilter filter = new DeviceTypeFilter(); - filter.setDeviceType("default"); + filter.setDeviceTypes(List.of("default")); filter.setDeviceNameFilter(""); loginSysAdmin(); EntityCountQuery countQuery = new EntityCountQuery(filter); + countByQueryAndCheck(countQuery, 97); - Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(97, count.longValue()); - - filter.setDeviceType("unknown"); - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(0, count.longValue()); - - filter.setDeviceType("default"); - filter.setDeviceNameFilter("Device1"); + filter.setDeviceTypes(List.of("unknown")); + countByQueryAndCheck(countQuery, 0); - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(11, count.longValue()); + filter.setDeviceTypes(List.of("default")); + filter.setDeviceNameFilter(devicePrefix + "1"); + countByQueryAndCheck(countQuery, 11); EntityListFilter entityListFilter = new EntityListFilter(); entityListFilter.setEntityType(EntityType.DEVICE); entityListFilter.setEntityList(devices.stream().map(Device::getId).map(DeviceId::toString).collect(Collectors.toList())); countQuery = new EntityCountQuery(entityListFilter); + countByQueryAndCheck(countQuery, 97); - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(97, count.longValue()); - - Long count2 = doPostWithResponse("/api/entitiesQuery/count", query, Long.class); - Assert.assertEquals(initialCount + 97, count2.longValue()); + countByQueryAndCheck(countQuery, 97); } @Test @@ -371,11 +358,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - PageData data = - doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); - - Assert.assertEquals(97, data.getTotalElements()); + PageData data = findByQueryAndCheck(query, 97); Assert.assertEquals(10, data.getTotalPages()); Assert.assertTrue(data.hasNext()); Assert.assertEquals(10, data.getData().size()); @@ -383,8 +366,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(97, loadedEntities.size()); @@ -406,8 +388,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { pageLink = new EntityDataPageLink(10, 0, "device1", sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + data = findByQuery(query); Assert.assertEquals(11, data.getTotalElements()); Assert.assertEquals("Device19", data.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); @@ -423,9 +404,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityDataQuery query2 = new EntityDataQuery(filter2, pageLink2, entityFields2, null, null); - PageData data2 = - doPostWithTypedResponse("/api/entitiesQuery/find", query2, new TypeReference>() { - }); + PageData data2 = findByQuery(query2); Assert.assertEquals(97, data2.getTotalElements()); Assert.assertEquals(10, data2.getTotalPages()); @@ -473,20 +452,15 @@ public class EntityQueryControllerTest extends AbstractControllerTest { List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - - PageData data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {}); - - Assert.assertEquals(87, data.getTotalElements()); + findByQueryAndCheck(query, 87); filter.setFilters(List.of(new RelationEntityTypeFilter("NOT_CONTAINS", List.of(EntityType.DEVICE), false))); query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {}); - Assert.assertEquals(10, data.getTotalElements()); + findByQueryAndCheck(query, 10); filter.setFilters(List.of(new RelationEntityTypeFilter("NOT_CONTAINS", List.of(EntityType.DEVICE), true))); query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {}); - Assert.assertEquals(87, data.getTotalElements()); + findByQueryAndCheck(query, 87); } private EntityRelation createFromRelation(Device mainDevice, Device device, String relationType) { @@ -531,14 +505,12 @@ public class EntityQueryControllerTest extends AbstractControllerTest { List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + PageData data = findByQueryAndCheck(query, 67); List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(67, loadedEntities.size()); @@ -551,6 +523,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = new KeyFilter(); highTemperatureFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); + highTemperatureFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate predicate = new NumericFilterPredicate(); predicate.setValue(FilterPredicateValue.fromDouble(45)); predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); @@ -559,13 +532,11 @@ public class EntityQueryControllerTest extends AbstractControllerTest { query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + data = findByQuery(query); loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); @@ -604,6 +575,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { KeyFilter highTemperatureFilter = new KeyFilter(); highTemperatureFilter.setKey(new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, "alarmActiveTime")); + highTemperatureFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate predicate = new NumericFilterPredicate(); DynamicValue dynamicValue = @@ -627,16 +599,16 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - Awaitility.await() + await() .alias("data by query") .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> { - var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() {}); + var data = findByQuery(query); var loadedEntities = new ArrayList<>(data.getData()); return loadedEntities.size() == numOfDevices; }); - var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() {}); + var data = findByQuery(query); var loadedEntities = new ArrayList<>(data.getData()); Assert.assertEquals(numOfDevices, loadedEntities.size()); @@ -694,11 +666,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityDataQuery query = new EntityDataQuery(entityTypeFilter, pageLink, entityFields, null, null); - PageData data = - doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); - - Assert.assertEquals(97, data.getTotalElements()); + PageData data = findByQueryAndCheck(query, 97); Assert.assertEquals(10, data.getTotalPages()); Assert.assertTrue(data.hasNext()); Assert.assertEquals(10, data.getData().size()); @@ -712,9 +680,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { }); EntityCountQuery countQuery = new EntityCountQuery(entityTypeFilter); - - Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(97, count.longValue()); + countByQueryAndCheck(countQuery, 97); } @Test @@ -742,28 +708,29 @@ public class EntityQueryControllerTest extends AbstractControllerTest { KeyFilter activeAlarmTimeToLongFilter = getServerAttributeNumericGreaterThanKeyFilter("alarmActiveTime", 30); KeyFilter tenantOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", TEST_TENANT_NAME); KeyFilter wrongOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", "wrongName"); - KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT"); + KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT"); KeyFilter customerOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "CUSTOMER"); // all devices with ownerName = TEST TENANT - EntityCountQuery query = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerNameFilter)); - checkEntitiesCount(query, numOfDevices); + EntityCountQuery query = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerNameFilter)); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> countByQuery(query), + result -> result == numOfDevices); // all devices with ownerName = TEST TENANT - EntityCountQuery activeAlarmTimeToLongQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeToLongFilter, tenantOwnerNameFilter)); - checkEntitiesCount(activeAlarmTimeToLongQuery, 0); + EntityCountQuery activeAlarmTimeToLongQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeToLongFilter, tenantOwnerNameFilter)); + countByQueryAndCheck(activeAlarmTimeToLongQuery, 0); // all devices with wrong ownerName EntityCountQuery wrongTenantNameQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, wrongOwnerNameFilter)); - checkEntitiesCount(wrongTenantNameQuery, 0); + countByQueryAndCheck(wrongTenantNameQuery, 0); // all devices with owner type = TENANT EntityCountQuery tenantEntitiesQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerTypeFilter)); - checkEntitiesCount(tenantEntitiesQuery, numOfDevices); + countByQueryAndCheck(tenantEntitiesQuery, numOfDevices); // all devices with owner type = CUSTOMER EntityCountQuery customerEntitiesQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, customerOwnerTypeFilter)); - checkEntitiesCount(customerEntitiesQuery, 0); + countByQueryAndCheck(customerEntitiesQuery, 0); } @Test @@ -790,7 +757,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { KeyFilter activeAlarmTimeFilter = getServerAttributeNumericGreaterThanKeyFilter("alarmActiveTime", 5); KeyFilter tenantOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", TEST_TENANT_NAME); KeyFilter wrongOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", "wrongName"); - KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT"); + KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT"); KeyFilter customerOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "CUSTOMER"); EntityDataSortOrder sortOrder = new EntityDataSortOrder( @@ -851,41 +818,29 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - PageData data = - doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); - - Assert.assertEquals(1, data.getTotalElements()); - Assert.assertEquals(1, data.getTotalPages()); - Assert.assertEquals(1, data.getData().size()); + findByQueryAndCheck(query, 1); // unnassign dashboard login(TENANT_EMAIL, TENANT_PASSWORD); doDelete("/api/customer/" + savedCustomer.getId().getId().toString() + "/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class); login(CUSTOMER_USER_EMAIL, CUSTOMER_USER_PASSWORD); - PageData dataAfterUnassign = - doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); - - Assert.assertEquals(0, dataAfterUnassign.getTotalElements()); - Assert.assertEquals(0, dataAfterUnassign.getTotalPages()); - Assert.assertEquals(0, dataAfterUnassign.getData().size()); + findByQueryAndCheck(query, 0); } private void checkEntitiesByQuery(EntityDataQuery query, int expectedNumOfDevices, String expectedOwnerName, String expectedOwnerType) throws Exception { - Awaitility.await() + await() .alias("data by query") .atMost(30, TimeUnit.SECONDS) .until(() -> { - var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() {}); + var data = findByQuery(query); var loadedEntities = new ArrayList<>(data.getData()); return loadedEntities.size() == expectedNumOfDevices; }); - if (expectedNumOfDevices == 0) { - return; - } - var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() {}); + if (expectedNumOfDevices == 0) { + return; + } + var data = findByQuery(query); var loadedEntities = new ArrayList<>(data.getData()); Assert.assertEquals(expectedNumOfDevices, loadedEntities.size()); @@ -898,25 +853,37 @@ public class EntityQueryControllerTest extends AbstractControllerTest { String alarmActiveTime = entity.getLatest().get(EntityKeyType.ATTRIBUTE).getOrDefault("alarmActiveTime", new TsValue(0, "-1")).getValue(); Assert.assertEquals("Device" + i, name); - Assert.assertEquals( expectedOwnerName, ownerName); - Assert.assertEquals( expectedOwnerType, ownerType); + Assert.assertEquals(expectedOwnerName, ownerName); + Assert.assertEquals(expectedOwnerType, ownerType); Assert.assertEquals("1" + i, alarmActiveTime); } } - private void checkEntitiesCount(EntityCountQuery query, int expectedNumOfDevices) { - Awaitility.await() - .alias("count by query") - .atMost(30, TimeUnit.SECONDS) - .until(() -> { - var count = doPost("/api/entitiesQuery/count", query, Integer.class); - return count == expectedNumOfDevices; - }); - } + protected PageData findByQuery(EntityDataQuery query) throws Exception { + return doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() { + }); + } + + protected PageData findByQueryAndCheck(EntityDataQuery query, int expectedResultSize) throws Exception { + PageData result = findByQuery(query); + assertThat(result.getTotalElements()).isEqualTo(expectedResultSize); + return result; + } + + protected Long countByQuery(EntityCountQuery countQuery) throws Exception { + return doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); + } + + protected Long countByQueryAndCheck(EntityCountQuery query, long expectedResult) throws Exception { + Long result = countByQuery(query); + assertThat(result).isEqualTo(expectedResult); + return result; + } private KeyFilter getEntityFieldStringEqualToKeyFilter(String keyName, String value) { KeyFilter tenantOwnerNameFilter = new KeyFilter(); tenantOwnerNameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, keyName)); + tenantOwnerNameFilter.setValueType(EntityKeyValueType.STRING); StringFilterPredicate ownerNamePredicate = new StringFilterPredicate(); ownerNamePredicate.setValue(FilterPredicateValue.fromString(value)); ownerNamePredicate.setOperation(StringFilterPredicate.StringOperation.EQUAL); @@ -927,6 +894,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { private KeyFilter getServerAttributeNumericGreaterThanKeyFilter(String attribute, int value) { KeyFilter numericFilter = new KeyFilter(); numericFilter.setKey(new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, attribute)); + numericFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate predicate = new NumericFilterPredicate(); predicate.setValue(FilterPredicateValue.fromDouble(value)); predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); diff --git a/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java index c4074f0d00..0d7b81370a 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.io.IOUtils; import org.junit.After; import org.junit.Assert; @@ -27,7 +28,10 @@ import org.springframework.http.HttpHeaders; import org.springframework.mock.web.MockHttpServletResponse; import org.springframework.test.web.servlet.ResultActions; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.DashboardInfo; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TbResource; @@ -40,13 +44,16 @@ import org.thingsboard.server.common.data.lwm2m.LwM2mObject; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; +import org.thingsboard.server.common.data.widget.WidgetTypeInfo; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DaoSqlTest; import java.util.ArrayList; import java.util.Base64; import java.util.Collections; +import java.util.HashMap; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; @@ -63,7 +70,6 @@ public class TbResourceControllerTest extends AbstractControllerTest { private static final String JS_TEST_FILE_NAME = "test.js"; private static final String TEST_DATA = "77u/PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPCEtLQpGSUxFIElORk9STUFUSU9OCgpPTUEgUGVybWFuZW50IERvY3VtZW50CiAgIEZpbGU6IE9NQS1TVVAtTHdNMk1fQmluYXJ5QXBwRGF0YUNvbnRhaW5lci1WMV8wXzEtMjAxOTAyMjEtQQogICBUeXBlOiB4bWwKClB1YmxpYyBSZWFjaGFibGUgSW5mb3JtYXRpb24KICAgUGF0aDogaHR0cDovL3d3dy5vcGVubW9iaWxlYWxsaWFuY2Uub3JnL3RlY2gvcHJvZmlsZXMKICAgTmFtZTogTHdNMk1fQmluYXJ5QXBwRGF0YUNvbnRhaW5lci12MV8wXzEueG1sCgpOT1JNQVRJVkUgSU5GT1JNQVRJT04KCiAgSW5mb3JtYXRpb24gYWJvdXQgdGhpcyBmaWxlIGNhbiBiZSBmb3VuZCBpbiB0aGUgbGF0ZXN0IHJldmlzaW9uIG9mCgogIE9NQS1UUy1MV00yTV9CaW5hcnlBcHBEYXRhQ29udGFpbmVyLVYxXzBfMQoKICBUaGlzIGlzIGF2YWlsYWJsZSBhdCBodHRwOi8vd3d3Lm9wZW5tb2JpbGVhbGxpYW5jZS5vcmcvCgogIFNlbmQgY29tbWVudHMgdG8gaHR0cHM6Ly9naXRodWIuY29tL09wZW5Nb2JpbGVBbGxpYW5jZS9PTUFfTHdNMk1fZm9yX0RldmVsb3BlcnMvaXNzdWVzCgpDSEFOR0UgSElTVE9SWQoKMTUwNjIwMTggU3RhdHVzIGNoYW5nZWQgdG8gQXBwcm92ZWQgYnkgRE0sIERvYyBSZWYgIyBPTUEtRE0mU0UtMjAxOC0wMDYxLUlOUF9MV00yTV9BUFBEQVRBX1YxXzBfRVJQX2Zvcl9maW5hbF9BcHByb3ZhbAoyMTAyMjAxOSBTdGF0dXMgY2hhbmdlZCB0byBBcHByb3ZlZCBieSBJUFNPLCBEb2MgUmVmICMgT01BLUlQU08tMjAxOS0wMDI1LUlOUF9Md00yTV9PYmplY3RfQXBwX0RhdGFfQ29udGFpbmVyXzFfMF8xX2Zvcl9GaW5hbF9BcHByb3ZhbAoKTEVHQUwgRElTQ0xBSU1FUgoKQ29weXJpZ2h0IDIwMTkgT3BlbiBNb2JpbGUgQWxsaWFuY2UuCgpSZWRpc3RyaWJ1dGlvbiBhbmQgdXNlIGluIHNvdXJjZSBhbmQgYmluYXJ5IGZvcm1zLCB3aXRoIG9yIHdpdGhvdXQKbW9kaWZpY2F0aW9uLCBhcmUgcGVybWl0dGVkIHByb3ZpZGVkIHRoYXQgdGhlIGZvbGxvd2luZyBjb25kaXRpb25zCmFyZSBtZXQ6CgoxLiBSZWRpc3RyaWJ1dGlvbnMgb2Ygc291cmNlIGNvZGUgbXVzdCByZXRhaW4gdGhlIGFib3ZlIGNvcHlyaWdodApub3RpY2UsIHRoaXMgbGlzdCBvZiBjb25kaXRpb25zIGFuZCB0aGUgZm9sbG93aW5nIGRpc2NsYWltZXIuCjIuIFJlZGlzdHJpYnV0aW9ucyBpbiBiaW5hcnkgZm9ybSBtdXN0IHJlcHJvZHVjZSB0aGUgYWJvdmUgY29weXJpZ2h0Cm5vdGljZSwgdGhpcyBsaXN0IG9mIGNvbmRpdGlvbnMgYW5kIHRoZSBmb2xsb3dpbmcgZGlzY2xhaW1lciBpbiB0aGUKZG9jdW1lbnRhdGlvbiBhbmQvb3Igb3RoZXIgbWF0ZXJpYWxzIHByb3ZpZGVkIHdpdGggdGhlIGRpc3RyaWJ1dGlvbi4KMy4gTmVpdGhlciB0aGUgbmFtZSBvZiB0aGUgY29weXJpZ2h0IGhvbGRlciBub3IgdGhlIG5hbWVzIG9mIGl0cwpjb250cmlidXRvcnMgbWF5IGJlIHVzZWQgdG8gZW5kb3JzZSBvciBwcm9tb3RlIHByb2R1Y3RzIGRlcml2ZWQKZnJvbSB0aGlzIHNvZnR3YXJlIHdpdGhvdXQgc3BlY2lmaWMgcHJpb3Igd3JpdHRlbiBwZXJtaXNzaW9uLgoKVEhJUyBTT0ZUV0FSRSBJUyBQUk9WSURFRCBCWSBUSEUgQ09QWVJJR0hUIEhPTERFUlMgQU5EIENPTlRSSUJVVE9SUwoiQVMgSVMiIEFORCBBTlkgRVhQUkVTUyBPUiBJTVBMSUVEIFdBUlJBTlRJRVMsIElOQ0xVRElORywgQlVUIE5PVApMSU1JVEVEIFRPLCBUSEUgSU1QTElFRCBXQVJSQU5USUVTIE9GIE1FUkNIQU5UQUJJTElUWSBBTkQgRklUTkVTUwpGT1IgQSBQQVJUSUNVTEFSIFBVUlBPU0UgQVJFIERJU0NMQUlNRUQuIElOIE5PIEVWRU5UIFNIQUxMIFRIRQpDT1BZUklHSFQgSE9MREVSIE9SIENPTlRSSUJVVE9SUyBCRSBMSUFCTEUgRk9SIEFOWSBESVJFQ1QsIElORElSRUNULApJTkNJREVOVEFMLCBTUEVDSUFMLCBFWEVNUExBUlksIE9SIENPTlNFUVVFTlRJQUwgREFNQUdFUyAoSU5DTFVESU5HLApCVVQgTk9UIExJTUlURUQgVE8sIFBST0NVUkVNRU5UIE9GIFNVQlNUSVRVVEUgR09PRFMgT1IgU0VSVklDRVM7CkxPU1MgT0YgVVNFLCBEQVRBLCBPUiBQUk9GSVRTOyBPUiBCVVNJTkVTUyBJTlRFUlJVUFRJT04pIEhPV0VWRVIKQ0FVU0VEIEFORCBPTiBBTlkgVEhFT1JZIE9GIExJQUJJTElUWSwgV0hFVEhFUiBJTiBDT05UUkFDVCwgU1RSSUNUCkxJQUJJTElUWSwgT1IgVE9SVCAoSU5DTFVESU5HIE5FR0xJR0VOQ0UgT1IgT1RIRVJXSVNFKSBBUklTSU5HIElOCkFOWSBXQVkgT1VUIE9GIFRIRSBVU0UgT0YgVEhJUyBTT0ZUV0FSRSwgRVZFTiBJRiBBRFZJU0VEIE9GIFRIRQpQT1NTSUJJTElUWSBPRiBTVUNIIERBTUFHRS4KClRoZSBhYm92ZSBsaWNlbnNlIGlzIHVzZWQgYXMgYSBsaWNlbnNlIHVuZGVyIGNvcHlyaWdodCBvbmx5LiBQbGVhc2UKcmVmZXJlbmNlIHRoZSBPTUEgSVBSIFBvbGljeSBmb3IgcGF0ZW50IGxpY2Vuc2luZyB0ZXJtczoKaHR0cHM6Ly93d3cub21hc3BlY3dvcmtzLm9yZy9hYm91dC9pbnRlbGxlY3R1YWwtcHJvcGVydHktcmlnaHRzLwoKLS0+CjxMV00yTSB4bWxuczp4c2k9Imh0dHA6Ly93d3cudzMub3JnLzIwMDEvWE1MU2NoZW1hLWluc3RhbmNlIiB4c2k6bm9OYW1lc3BhY2VTY2hlbWFMb2NhdGlvbj0iaHR0cDovL29wZW5tb2JpbGVhbGxpYW5jZS5vcmcvdGVjaC9wcm9maWxlcy9MV00yTS54c2QiPgoJPE9iamVjdCBPYmplY3RUeXBlPSJNT0RlZmluaXRpb24iPgoJCTxOYW1lPkJpbmFyeUFwcERhdGFDb250YWluZXI8L05hbWU+CgkJPERlc2NyaXB0aW9uMT48IVtDREFUQVtUaGlzIEx3TTJNIE9iamVjdHMgcHJvdmlkZXMgdGhlIGFwcGxpY2F0aW9uIHNlcnZpY2UgZGF0YSByZWxhdGVkIHRvIGEgTHdNMk0gU2VydmVyLCBlZy4gV2F0ZXIgbWV0ZXIgZGF0YS4gClRoZXJlIGFyZSBzZXZlcmFsIG1ldGhvZHMgdG8gY3JlYXRlIGluc3RhbmNlIHRvIGluZGljYXRlIHRoZSBtZXNzYWdlIGRpcmVjdGlvbiBiYXNlZCBvbiB0aGUgbmVnb3RpYXRpb24gYmV0d2VlbiBBcHBsaWNhdGlvbiBhbmQgTHdNMk0uIFRoZSBDbGllbnQgYW5kIFNlcnZlciBzaG91bGQgbmVnb3RpYXRlIHRoZSBpbnN0YW5jZShzKSB1c2VkIHRvIGV4Y2hhbmdlIHRoZSBkYXRhLiBGb3IgZXhhbXBsZToKIC0gVXNpbmcgYSBzaW5nbGUgaW5zdGFuY2UgZm9yIGJvdGggZGlyZWN0aW9ucyBjb21tdW5pY2F0aW9uLCBmcm9tIENsaWVudCB0byBTZXJ2ZXIgYW5kIGZyb20gU2VydmVyIHRvIENsaWVudC4KIC0gVXNpbmcgYW4gaW5zdGFuY2UgZm9yIGNvbW11bmljYXRpb24gZnJvbSBDbGllbnQgdG8gU2VydmVyIGFuZCBhbm90aGVyIG9uZSBmb3IgY29tbXVuaWNhdGlvbiBmcm9tIFNlcnZlciB0byBDbGllbnQKIC0gVXNpbmcgc2V2ZXJhbCBpbnN0YW5jZXMKXV0+PC9EZXNjcmlwdGlvbjE+CgkJPE9iamVjdElEPjE5PC9PYmplY3RJRD4KCQk8T2JqZWN0VVJOPnVybjpvbWE6bHdtMm06b21hOjE5PC9PYmplY3RVUk4+CgkJPExXTTJNVmVyc2lvbj4xLjA8L0xXTTJNVmVyc2lvbj4KCQk8T2JqZWN0VmVyc2lvbj4xLjA8L09iamVjdFZlcnNpb24+CgkJPE11bHRpcGxlSW5zdGFuY2VzPk11bHRpcGxlPC9NdWx0aXBsZUluc3RhbmNlcz4KCQk8TWFuZGF0b3J5Pk9wdGlvbmFsPC9NYW5kYXRvcnk+CgkJPFJlc291cmNlcz4KCQkJPEl0ZW0gSUQ9IjAiPjxOYW1lPkRhdGE8L05hbWU+CgkJCQk8T3BlcmF0aW9ucz5SVzwvT3BlcmF0aW9ucz4KCQkJCTxNdWx0aXBsZUluc3RhbmNlcz5NdWx0aXBsZTwvTXVsdGlwbGVJbnN0YW5jZXM+CgkJCQk8TWFuZGF0b3J5Pk1hbmRhdG9yeTwvTWFuZGF0b3J5PgoJCQkJPFR5cGU+T3BhcXVlPC9UeXBlPgoJCQkJPFJhbmdlRW51bWVyYXRpb24gLz4KCQkJCTxVbml0cyAvPgoJCQkJPERlc2NyaXB0aW9uPjwhW0NEQVRBW0luZGljYXRlcyB0aGUgYXBwbGljYXRpb24gZGF0YSBjb250ZW50Ll1dPjwvRGVzY3JpcHRpb24+CgkJCTwvSXRlbT4KCQkJPEl0ZW0gSUQ9IjEiPjxOYW1lPkRhdGEgUHJpb3JpdHk8L05hbWU+CgkJCQk8T3BlcmF0aW9ucz5SVzwvT3BlcmF0aW9ucz4KCQkJCTxNdWx0aXBsZUluc3RhbmNlcz5TaW5nbGU8L011bHRpcGxlSW5zdGFuY2VzPgoJCQkJPE1hbmRhdG9yeT5PcHRpb25hbDwvTWFuZGF0b3J5PgoJCQkJPFR5cGU+SW50ZWdlcjwvVHlwZT4KCQkJCTxSYW5nZUVudW1lcmF0aW9uPjEgYnl0ZXM8L1JhbmdlRW51bWVyYXRpb24+CgkJCQk8VW5pdHMgLz4KCQkJCTxEZXNjcmlwdGlvbj48IVtDREFUQVtJbmRpY2F0ZXMgdGhlIEFwcGxpY2F0aW9uIGRhdGEgcHJpb3JpdHk6CjA6SW1tZWRpYXRlCjE6QmVzdEVmZm9ydAoyOkxhdGVzdAozLTEwMDogUmVzZXJ2ZWQgZm9yIGZ1dHVyZSB1c2UuCjEwMS0yNTQ6IFByb3ByaWV0YXJ5IG1vZGUuXV0+PC9EZXNjcmlwdGlvbj4KCQkJPC9JdGVtPgoJCQk8SXRlbSBJRD0iMiI+PE5hbWU+RGF0YSBDcmVhdGlvbiBUaW1lPC9OYW1lPgoJCQkJPE9wZXJhdGlvbnM+Ulc8L09wZXJhdGlvbnM+CgkJCQk8TXVsdGlwbGVJbnN0YW5jZXM+U2luZ2xlPC9NdWx0aXBsZUluc3RhbmNlcz4KCQkJCTxNYW5kYXRvcnk+T3B0aW9uYWw8L01hbmRhdG9yeT4KCQkJCTxUeXBlPlRpbWU8L1R5cGU+CgkJCQk8UmFuZ2VFbnVtZXJhdGlvbiAvPgoJCQkJPFVuaXRzIC8+CgkJCQk8RGVzY3JpcHRpb24+PCFbQ0RBVEFbSW5kaWNhdGVzIHRoZSBEYXRhIGluc3RhbmNlIGNyZWF0aW9uIHRpbWVzdGFtcC5dXT48L0Rlc2NyaXB0aW9uPgoJCQk8L0l0ZW0+CgkJCTxJdGVtIElEPSIzIj48TmFtZT5EYXRhIERlc2NyaXB0aW9uPC9OYW1lPgoJCQkJPE9wZXJhdGlvbnM+Ulc8L09wZXJhdGlvbnM+CgkJCQk8TXVsdGlwbGVJbnN0YW5jZXM+U2luZ2xlPC9NdWx0aXBsZUluc3RhbmNlcz4KCQkJCTxNYW5kYXRvcnk+T3B0aW9uYWw8L01hbmRhdG9yeT4KCQkJCTxUeXBlPlN0cmluZzwvVHlwZT4KCQkJCTxSYW5nZUVudW1lcmF0aW9uPjMyIGJ5dGVzPC9SYW5nZUVudW1lcmF0aW9uPgoJCQkJPFVuaXRzIC8+CgkJCQk8RGVzY3JpcHRpb24+PCFbQ0RBVEFbSW5kaWNhdGVzIHRoZSBkYXRhIGRlc2NyaXB0aW9uLgplLmcuICJtZXRlciByZWFkaW5nIi5dXT48L0Rlc2NyaXB0aW9uPgoJCQk8L0l0ZW0+CgkJCTxJdGVtIElEPSI0Ij48TmFtZT5EYXRhIEZvcm1hdDwvTmFtZT4KCQkJCTxPcGVyYXRpb25zPlJXPC9PcGVyYXRpb25zPgoJCQkJPE11bHRpcGxlSW5zdGFuY2VzPlNpbmdsZTwvTXVsdGlwbGVJbnN0YW5jZXM+CgkJCQk8TWFuZGF0b3J5Pk9wdGlvbmFsPC9NYW5kYXRvcnk+CgkJCQk8VHlwZT5TdHJpbmc8L1R5cGU+CgkJCQk8UmFuZ2VFbnVtZXJhdGlvbj4zMiBieXRlczwvUmFuZ2VFbnVtZXJhdGlvbj4KCQkJCTxVbml0cyAvPgoJCQkJPERlc2NyaXB0aW9uPjwhW0NEQVRBW0luZGljYXRlcyB0aGUgZm9ybWF0IG9mIHRoZSBBcHBsaWNhdGlvbiBEYXRhLgplLmcuIFlHLU1ldGVyLVdhdGVyLVJlYWRpbmcKVVRGOC1zdHJpbmcKXV0+PC9EZXNjcmlwdGlvbj4KCQkJPC9JdGVtPgoJCQk8SXRlbSBJRD0iNSI+PE5hbWU+QXBwIElEPC9OYW1lPgoJCQkJPE9wZXJhdGlvbnM+Ulc8L09wZXJhdGlvbnM+CgkJCQk8TXVsdGlwbGVJbnN0YW5jZXM+U2luZ2xlPC9NdWx0aXBsZUluc3RhbmNlcz4KCQkJCTxNYW5kYXRvcnk+T3B0aW9uYWw8L01hbmRhdG9yeT4KCQkJCTxUeXBlPkludGVnZXI8L1R5cGU+CgkJCQk8UmFuZ2VFbnVtZXJhdGlvbj4yIGJ5dGVzPC9SYW5nZUVudW1lcmF0aW9uPgoJCQkJPFVuaXRzIC8+CgkJCQk8RGVzY3JpcHRpb24+PCFbQ0RBVEFbSW5kaWNhdGVzIHRoZSBkZXN0aW5hdGlvbiBBcHBsaWNhdGlvbiBJRC5dXT48L0Rlc2NyaXB0aW9uPgoJCQk8L0l0ZW0+PC9SZXNvdXJjZXM+CgkJPERlc2NyaXB0aW9uMj48IVtDREFUQVtdXT48L0Rlc2NyaXB0aW9uMj4KCTwvT2JqZWN0Pgo8L0xXTTJNPgo="; - private Tenant savedTenant; private User tenantAdmin; @@ -222,28 +228,196 @@ public class TbResourceControllerTest extends AbstractControllerTest { } @Test - public void testShoudNotDeleteTbResourceIfAssignedToWidgetType() throws Exception { + public void testUnForcedDeleteTbResourceIfAssignedToWidgetType() throws Exception { TbResource resource = new TbResource(); - resource.setResourceType(ResourceType.JKS); + resource.setResourceType(ResourceType.JS_MODULE); resource.setTitle("My first resource"); - resource.setFileName(DEFAULT_FILE_NAME); + resource.setFileName(JS_TEST_FILE_NAME); + resource.setTenantId(savedTenant.getId()); resource.setEncodedData(TEST_DATA); + resource.setResourceKey(JS_TEST_FILE_NAME); TbResourceInfo savedResource = save(resource); - Mockito.reset(tbClusterService, auditLogService); - String resourceIdStr = savedResource.getId().getId().toString(); + var link = resource.getLink(); + WidgetTypeDetails widgetType = new WidgetTypeDetails(); + widgetType.setName("Widget Type"); + widgetType.setTenantId(savedTenant.getId()); + widgetType.setDescriptor(JacksonUtil.newObjectNode() + .put("controllerScript", "self.onInit = function() {\n self.ctx.$scope.actionWidget.onInit();\n}\n\nself.typeParameters = function() {\n return {\n previewWidth: '300px',\n previewHeight: '320px',\n embedTitlePanel: true,\n targetDeviceOptional: true,\n displayRpcMessageToast: false\n };\n};\n\nself.onDestroy = function() {\n}") + .put("settingsSchema", "") + .put("dataKeySettingsSchema", "{}\n") + .put("settingsDirective", "tb-scada-symbol-widget-settings") + .put("hasBasicMode", true) + .put("basicModeDirective", "tb-scada-symbol-basic-config") + .put("resource", link)); + WidgetType savedWidgetType = doPost("/api/widgetType", widgetType, WidgetTypeDetails.class); + + var deleteResponse = doDelete("/api/resource/" + savedResource.getUuidId() + "?force=false") + .andExpect(status().isBadRequest()) + .andReturn() + .getResponse() + .getContentAsString(); + + Assert.assertNotNull(deleteResponse); + + boolean isSuccess = JacksonUtil.toJsonNode(deleteResponse).get("success").asBoolean(); + Assert.assertFalse(isSuccess); + + var referenceValues = JacksonUtil.toJsonNode(deleteResponse).get("references"); + Assert.assertNotNull(referenceValues); + + var widgetTypeInfos = JacksonUtil.readValue(referenceValues.toString(), new TypeReference>>() { + }); + Assert.assertNotNull(widgetTypeInfos); + Assert.assertFalse(widgetTypeInfos.isEmpty()); + Assert.assertEquals(1, widgetTypeInfos.size()); + + var dashboardInfo = widgetTypeInfos.get(EntityType.WIDGET_TYPE.name()).get(0); + Assert.assertNotNull(dashboardInfo); - //create widget type + WidgetTypeInfo foundedWidgetType = doGet("/api/widgetTypeInfo/" + savedWidgetType.getId().getId().toString(), WidgetTypeInfo.class); + Assert.assertNotNull(foundedWidgetType); + Assert.assertEquals(foundedWidgetType, dashboardInfo); + } + + @Test + public void testForcedDeleteTbResourceIfAssignedToWidgetType() throws Exception { + TbResource resource = new TbResource(); + resource.setResourceType(ResourceType.JS_MODULE); + resource.setTitle("My first resource"); + resource.setFileName(JS_TEST_FILE_NAME); + resource.setTenantId(savedTenant.getId()); + resource.setEncodedData(TEST_DATA); + resource.setResourceKey(JS_TEST_FILE_NAME); + TbResourceInfo savedResource = save(resource); + + var link = resource.getLink(); WidgetTypeDetails widgetType = new WidgetTypeDetails(); widgetType.setName("Widget Type"); - widgetType.setDescriptor(JacksonUtil.fromString(String.format("{ \"resources\": [{\"url\":\"tb-resource;/api/resource/jks/tenant/%s\",\"isModule\":true}]}", savedResource.getResourceKey()), JsonNode.class)); + widgetType.setTenantId(savedTenant.getId()); + widgetType.setDescriptor(JacksonUtil.newObjectNode() + .put("controllerScript", "self.onInit = function() {\n self.ctx.$scope.actionWidget.onInit();\n}\n\nself.typeParameters = function() {\n return {\n previewWidth: '300px',\n previewHeight: '320px',\n embedTitlePanel: true,\n targetDeviceOptional: true,\n displayRpcMessageToast: false\n };\n};\n\nself.onDestroy = function() {\n}") + .put("settingsSchema", "") + .put("dataKeySettingsSchema", "{}\n") + .put("settingsDirective", "tb-scada-symbol-widget-settings") + .put("hasBasicMode", true) + .put("basicModeDirective", "tb-scada-symbol-basic-config") + .put("resource", link)); doPost("/api/widgetType", widgetType, WidgetTypeDetails.class); - doDelete("/api/resource/" + resourceIdStr) + var deleteResponse = doDelete("/api/resource/" + savedResource.getUuidId() + "?force=true") + .andExpect(status().isOk()) + .andReturn() + .getResponse() + .getContentAsString(); + + Assert.assertNotNull(deleteResponse); + + boolean isSuccess = JacksonUtil.toJsonNode(deleteResponse).get("success").asBoolean(); + Assert.assertTrue(isSuccess); + + var referenceValues = JacksonUtil.toJsonNode(deleteResponse).get("references"); + var widgetTypeInfos = JacksonUtil.readValue(referenceValues.toString(), new TypeReference>>() { + }); + Assert.assertNull(widgetTypeInfos); + } + + @Test + public void testUnForcedDeleteTbResourceIfAssignedToDashboard() throws Exception { + TbResource resource = new TbResource(); + resource.setResourceType(ResourceType.JS_MODULE); + resource.setTitle("My first resource"); + resource.setFileName(JS_TEST_FILE_NAME); + resource.setTenantId(savedTenant.getId()); + resource.setEncodedData(TEST_DATA); + resource.setResourceKey(JS_TEST_FILE_NAME); + TbResourceInfo savedResource = save(resource); + + var link = resource.getLink(); + Dashboard dashboard = new Dashboard(); + dashboard.setTitle("My dashboard"); + dashboard.setTenantId(savedTenant.getId()); + dashboard.setConfiguration(JacksonUtil.newObjectNode() + .set("widgets", JacksonUtil.toJsonNode(""" + {"xxx": + {"config":{"actions":{"elementClick":[ + {"customResources":[{"url":{"entityType":"TB_RESOURCE","id": + "tb-resource;/api/resource/js_module/tenant/gateway-management-extension.js"},"isModule":true}, + {"url":"tb-resource;/api/resource/js_module/tenant/gateway-management-extension.js","isModule":true}]}]}}}} + """)) + .put("resource", link)); + Dashboard savedDashboard = doPost("/api/dashboard", dashboard, Dashboard.class); + + var deleteResponse = doDelete("/api/resource/" + savedResource.getUuidId() + "?force=false") .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Following widget types use this resource: " - + widgetType.getName()))); + .andReturn() + .getResponse() + .getContentAsString(); + + Assert.assertNotNull(deleteResponse); + + boolean isSuccess = JacksonUtil.toJsonNode(deleteResponse).get("success").asBoolean(); + Assert.assertFalse(isSuccess); + + var referenceValues = JacksonUtil.toJsonNode(deleteResponse).get("references"); + Assert.assertNotNull(referenceValues); + + var dashboardInfos = JacksonUtil.readValue(referenceValues.toString(), new TypeReference>>() { + }); + Assert.assertNotNull(dashboardInfos); + Assert.assertFalse(dashboardInfos.isEmpty()); + Assert.assertEquals(1, dashboardInfos.size()); + + var dashboardInfo = dashboardInfos.get(EntityType.DASHBOARD.name()).get(0); + Assert.assertNotNull(dashboardInfo); + + DashboardInfo foundDashboard = doGet("/api/dashboard/info/" + savedDashboard.getId().getId().toString(), DashboardInfo.class); + Assert.assertNotNull(foundDashboard); + Assert.assertEquals(foundDashboard, dashboardInfo); + } + + @Test + public void testForcedDeleteTbResourceIfAssignedToDashboard() throws Exception { + TbResource resource = new TbResource(); + resource.setResourceType(ResourceType.JS_MODULE); + resource.setTitle("My first resource"); + resource.setFileName(JS_TEST_FILE_NAME); + resource.setTenantId(savedTenant.getId()); + resource.setEncodedData(TEST_DATA); + resource.setResourceKey(JS_TEST_FILE_NAME); + TbResourceInfo savedResource = save(resource); + + var link = resource.getLink(); + Dashboard dashboard = new Dashboard(); + dashboard.setTitle("My dashboard"); + dashboard.setTenantId(savedTenant.getId()); + dashboard.setConfiguration(JacksonUtil.newObjectNode() + .set("widgets", JacksonUtil.toJsonNode(""" + {"xxx": + {"config":{"actions":{"elementClick":[ + {"customResources":[{"url":{"entityType":"TB_RESOURCE","id": + "tb-resource;/api/resource/js_module/tenant/gateway-management-extension.js"},"isModule":true}, + {"url":"tb-resource;/api/resource/js_module/tenant/gateway-management-extension.js","isModule":true}]}]}}}} + """)) + .put("resource", link)); + doPost("/api/dashboard", dashboard, Dashboard.class); + + var deleteResponse = doDelete("/api/resource/" + savedResource.getUuidId() + "?force=true") + .andExpect(status().isOk()) + .andReturn() + .getResponse() + .getContentAsString(); + + Assert.assertNotNull(deleteResponse); + + boolean isSuccess = JacksonUtil.toJsonNode(deleteResponse).get("success").asBoolean(); + Assert.assertTrue(isSuccess); + + var referenceValues = JacksonUtil.toJsonNode(deleteResponse).get("references"); + var dashboardInfos = JacksonUtil.readValue(referenceValues.toString(), new TypeReference>>() { + }); + Assert.assertNull(dashboardInfos); } @Test @@ -676,7 +850,8 @@ public class TbResourceControllerTest extends AbstractControllerTest { List resources = loadLwm2mResources(); List objects = - doGetTyped("/api/resource/lwm2m/page?pageSize=100&page=0", new TypeReference<>() {}); + doGetTyped("/api/resource/lwm2m/page?pageSize=100&page=0", new TypeReference<>() { + }); Assert.assertNotNull(objects); Assert.assertEquals(resources.size(), objects.size()); @@ -690,7 +865,8 @@ public class TbResourceControllerTest extends AbstractControllerTest { List resources = loadLwm2mResources(); List objects = - doGetTyped("/api/resource/lwm2m?sortProperty=id&sortOrder=ASC&objectIds=3_1.2,5_1.2,19_1.1", new TypeReference<>() {}); + doGetTyped("/api/resource/lwm2m?sortProperty=id&sortOrder=ASC&objectIds=3_1.2,5_1.2,19_1.1", new TypeReference<>() { + }); Assert.assertNotNull(objects); Assert.assertEquals(3, objects.size()); diff --git a/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java index 6d953c1722..0daa56728b 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.queue.TbQueueCallback; import java.util.ArrayList; import java.util.Collections; @@ -44,6 +45,7 @@ import java.util.List; import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -354,7 +356,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { argument -> argument.getClass().equals(TenantProfile.class); if (ComponentLifecycleEvent.DELETED.equals(event)) { Mockito.verify(tbClusterService, times(cntTime)).onTenantProfileDelete(Mockito.argThat(matcherTenantProfile), - Mockito.isNull()); + eq(TbQueueCallback.EMPTY)); testBroadcastEntityStateChangeEventNever(createEntityId_NULL_UUID(new Tenant())); } else { Mockito.verify(tbClusterService, times(cntTime)).onTenantProfileChange(Mockito.argThat(matcherTenantProfile), diff --git a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java index c33e9f7202..9801907d3b 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java @@ -83,7 +83,8 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @Slf4j @DaoSqlTest @TestPropertySource(properties = { - "server.ws.alarms_per_alarm_status_subscription_cache_size=5" + "server.ws.alarms_per_alarm_status_subscription_cache_size=5", + "server.ws.dynamic_page_link.refresh_interval=15" }) public class WebsocketApiTest extends AbstractControllerTest { @Autowired @@ -324,6 +325,83 @@ public class WebsocketApiTest extends AbstractControllerTest { Assert.assertEquals(1, update.getCount()); } + @Test + public void testAlarmCountWsCmdWithSingleEntityFilter() throws Exception { + loginTenantAdmin(); + + SingleEntityFilter singleEntityFilter = new SingleEntityFilter(); + singleEntityFilter.setSingleEntity(tenantId); + AlarmCountQuery alarmCountQuery = new AlarmCountQuery(singleEntityFilter); + AlarmCountCmd cmd1 = new AlarmCountCmd(1, alarmCountQuery); + + getWsClient().send(cmd1); + + AlarmCountUpdate update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); + Assert.assertEquals(1, update.getCmdId()); + Assert.assertEquals(0, update.getCount()); + + //create alarm, check count = 1 + getWsClient().registerWaitForUpdate(); + + Alarm alarm = new Alarm(); + alarm.setOriginator(tenantId); + alarm.setType("TEST ALARM"); + alarm.setSeverity(AlarmSeverity.WARNING); + alarm = doPost("/api/alarm", alarm, Alarm.class); + + update = getWsClient().parseAlarmCountReply(getWsClient().waitForUpdate()); + Assert.assertEquals(1, update.getCmdId()); + Assert.assertEquals(1, update.getCount()); + + // set wrong entity id in filter, check count = 0 + singleEntityFilter.setSingleEntity(tenantAdminUserId); + AlarmCountCmd cmd3 = new AlarmCountCmd(2, alarmCountQuery); + + getWsClient().send(cmd3); + + update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); + Assert.assertEquals(2, update.getCmdId()); + Assert.assertEquals(0, update.getCount()); + } + + @Test + public void testAlarmCountWsCmdWithDeviceType() throws Exception { + loginTenantAdmin(); + + DeviceTypeFilter deviceTypeFilter = new DeviceTypeFilter(); + deviceTypeFilter.setDeviceTypes(List.of("default")); + AlarmCountQuery alarmCountQuery = new AlarmCountQuery(deviceTypeFilter); + AlarmCountCmd cmd1 = new AlarmCountCmd(1, alarmCountQuery); + + getWsClient().send(cmd1); + + AlarmCountUpdate update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); + Assert.assertEquals(1, update.getCmdId()); + Assert.assertEquals(0, update.getCount()); + + getWsClient().registerWaitForUpdate(); + + Alarm alarm = new Alarm(); + alarm.setOriginator(device.getId()); + alarm.setType("TEST ALARM"); + alarm.setSeverity(AlarmSeverity.WARNING); + + alarm = doPost("/api/alarm", alarm, Alarm.class); + + update = getWsClient().parseAlarmCountReply(getWsClient().waitForUpdate()); + Assert.assertEquals(1, update.getCmdId()); + Assert.assertEquals(1, update.getCount()); + + deviceTypeFilter.setDeviceTypes(List.of("non-existing")); + AlarmCountCmd cmd3 = new AlarmCountCmd(3, alarmCountQuery); + + getWsClient().send(cmd3); + + update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); + Assert.assertEquals(3, update.getCmdId()); + Assert.assertEquals(0, update.getCount()); + } + @Test public void testAlarmStatusWsCmd() throws Exception { loginTenantAdmin(); @@ -372,17 +450,18 @@ public class WebsocketApiTest extends AbstractControllerTest { doPost("/api/alarm", alarm2, Alarm.class); - AlarmStatusUpdate alarmStatusUpdate3 = JacksonUtil.fromString(getWsClient().waitForReply(), AlarmStatusUpdate.class); + AlarmStatusUpdate alarmStatusUpdate3 = JacksonUtil.fromString(getWsClient().waitForUpdate(), AlarmStatusUpdate.class); Assert.assertEquals(1, alarmStatusUpdate3.getCmdId()); Assert.assertTrue(alarmStatusUpdate3.isActive()); //change severity + getWsClient().registerWaitForUpdate(); alarm2.setSeverity(AlarmSeverity.MAJOR); Alarm updatedAlarm = doPost("/api/alarm", alarm2, Alarm.class); Assert.assertNotNull(updatedAlarm); Assert.assertEquals(AlarmSeverity.MAJOR, updatedAlarm.getSeverity()); - AlarmStatusUpdate alarmStatusUpdate4 = JacksonUtil.fromString(getWsClient().waitForReply(), AlarmStatusUpdate.class); + AlarmStatusUpdate alarmStatusUpdate4 = JacksonUtil.fromString(getWsClient().waitForUpdate(), AlarmStatusUpdate.class); Assert.assertEquals(1, alarmStatusUpdate4.getCmdId()); Assert.assertFalse(alarmStatusUpdate4.isActive()); diff --git a/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java index 9212afa7c5..feac7adb04 100644 --- a/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java @@ -90,14 +90,12 @@ import org.thingsboard.server.gen.edge.v1.EdgeConfiguration; import org.thingsboard.server.gen.edge.v1.OAuth2ClientUpdateMsg; import org.thingsboard.server.gen.edge.v1.OAuth2DomainUpdateMsg; import org.thingsboard.server.gen.edge.v1.QueueUpdateMsg; -import org.thingsboard.server.gen.edge.v1.RuleChainMetadataRequestMsg; import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; import org.thingsboard.server.gen.edge.v1.SyncCompletedMsg; import org.thingsboard.server.gen.edge.v1.TenantProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.TenantUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.gen.edge.v1.UplinkMsg; import org.thingsboard.server.gen.edge.v1.UserCredentialsUpdateMsg; import org.thingsboard.server.gen.edge.v1.UserUpdateMsg; @@ -142,35 +140,14 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { installation(); edgeImitator = new EdgeImitator("localhost", 7070, edge.getRoutingKey(), edge.getSecret()); + edgeImitator.expectMessageAmount(25); edgeImitator.ignoreType(OAuth2ClientUpdateMsg.class); edgeImitator.ignoreType(OAuth2DomainUpdateMsg.class); - edgeImitator.expectMessageAmount(26); edgeImitator.connect(); - requestEdgeRuleChainMetadata(); - verifyEdgeConnectionAndInitialData(); } - private void requestEdgeRuleChainMetadata() throws Exception { - RuleChainId rootRuleChainId = getEdgeRootRuleChainId(); - RuleChainMetadataRequestMsg.Builder builder = RuleChainMetadataRequestMsg.newBuilder() - .setRuleChainIdMSB(rootRuleChainId.getId().getMostSignificantBits()) - .setRuleChainIdLSB(rootRuleChainId.getId().getLeastSignificantBits()); - testAutoGeneratedCodeByProtobuf(builder); - UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder() - .addRuleChainMetadataRequestMsg(builder.build()); - edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); - } - - private RuleChainId getEdgeRootRuleChainId() throws Exception { - return doGetTypedWithPageLink("/api/ruleChains?type={type}&", new TypeReference>() { - }, - new PageLink(100, 0, "Edge Root Rule Chain"), - "EDGE") - .getData().get(0).getId(); - } - @After public void teardownEdgeTest() { try { @@ -213,6 +190,19 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { doPost("/api/ruleChain/metadata", rootRuleChainMetadata, RuleChainMetaData.class); } + private RuleChainId getEdgeRootRuleChainId() throws Exception { + List edgeRuleChains = doGetTypedWithPageLink("/api/ruleChains?type={type}&", + new TypeReference>() {}, + new PageLink(100, 0, "Edge Root Rule Chain"), + "EDGE").getData(); + for (RuleChain edgeRuleChain : edgeRuleChains) { + if (edgeRuleChain.isRoot()) { + return edgeRuleChain.getId(); + } + } + throw new RuntimeException("Root rule chain not found"); + } + protected void extendDeviceProfileData(DeviceProfile deviceProfile) { DeviceProfileData profileData = deviceProfile.getProfileData(); List alarms = new ArrayList<>(); @@ -255,8 +245,8 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { validateMsgsCnt(RuleChainUpdateMsg.class, 1); UUID ruleChainUUID = validateRuleChains(); - // 1 from request message - validateMsgsCnt(RuleChainMetadataUpdateMsg.class, 2); + // 1 from rule chain fetcher + validateMsgsCnt(RuleChainMetadataUpdateMsg.class, 1); validateRuleChainMetadataUpdates(ruleChainUUID); // 4 messages ('general', 'mail', 'connectivity', 'jwt') @@ -438,12 +428,11 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { } private void validateRuleChainMetadataUpdates(UUID expectedRuleChainUUID) { - Optional ruleChainMetadataUpdateOpt = edgeImitator.findMessageByType(RuleChainMetadataUpdateMsg.class); - Assert.assertTrue(ruleChainMetadataUpdateOpt.isPresent()); - RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = ruleChainMetadataUpdateOpt.get(); + Optional ruleChainMetadataUpdateMsgOpt = edgeImitator.findMessageByType(RuleChainMetadataUpdateMsg.class); + Assert.assertTrue(ruleChainMetadataUpdateMsgOpt.isPresent()); + RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = ruleChainMetadataUpdateMsgOpt.get(); Assert.assertEquals(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, ruleChainMetadataUpdateMsg.getMsgType()); RuleChainMetaData ruleChainMetaData = JacksonUtil.fromString(ruleChainMetadataUpdateMsg.getEntity(), RuleChainMetaData.class, true); - Assert.assertNotNull(ruleChainMetaData); Assert.assertEquals(expectedRuleChainUUID, ruleChainMetaData.getRuleChainId().getId()); } diff --git a/application/src/test/java/org/thingsboard/server/edge/AssetEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/AssetEdgeTest.java index 79ec2f3d87..4d9840b936 100644 --- a/application/src/test/java/org/thingsboard/server/edge/AssetEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/AssetEdgeTest.java @@ -169,6 +169,7 @@ public class AssetEdgeTest extends AbstractEdgeTest { public void testSendAssetToCloud() throws Exception { Asset asset = buildAssetForUplinkMsg("Asset Edge 2"); + // created asset on edge UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); AssetUpdateMsg.Builder assetUpdateMsgBuilder = AssetUpdateMsg.newBuilder(); assetUpdateMsgBuilder.setIdMSB(asset.getUuidId().getMostSignificantBits()); @@ -191,6 +192,32 @@ public class AssetEdgeTest extends AbstractEdgeTest { Asset foundAsset = doGet("/api/asset/" + asset.getUuidId(), Asset.class); Assert.assertNotNull(foundAsset); Assert.assertEquals("Asset Edge 2", foundAsset.getName()); + + // update asset on edge + asset.setName("Asset Edge 2 Updated"); + + uplinkMsgBuilder = UplinkMsg.newBuilder(); + assetUpdateMsgBuilder = AssetUpdateMsg.newBuilder(); + assetUpdateMsgBuilder.setIdMSB(asset.getUuidId().getMostSignificantBits()); + assetUpdateMsgBuilder.setIdLSB(asset.getUuidId().getLeastSignificantBits()); + assetUpdateMsgBuilder.setEntity(JacksonUtil.toString(asset)); + assetUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE); + testAutoGeneratedCodeByProtobuf(assetUpdateMsgBuilder); + uplinkMsgBuilder.addAssetUpdateMsg(assetUpdateMsgBuilder.build()); + + testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder); + + edgeImitator.expectResponsesAmount(1); + edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); + + Assert.assertTrue(edgeImitator.waitForResponses()); + + latestResponseMsg = edgeImitator.getLatestResponseMsg(); + Assert.assertTrue(latestResponseMsg.getSuccess()); + + foundAsset = doGet("/api/asset/" + asset.getUuidId(), Asset.class); + Assert.assertNotNull(foundAsset); + Assert.assertEquals("Asset Edge 2 Updated", foundAsset.getName()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java index b848178c58..bbf3d17f0d 100644 --- a/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java @@ -184,6 +184,7 @@ public class DashboardEdgeTest extends AbstractEdgeTest { Dashboard dashboard = buildDashboardForUplinkMsg(savedCustomer); + // create dashboard on edge UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); DashboardUpdateMsg.Builder dashboardUpdateMsgBuilder = DashboardUpdateMsg.newBuilder(); dashboardUpdateMsgBuilder.setIdMSB(dashboard.getUuidId().getMostSignificantBits()); diff --git a/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java index a6cd127097..da8e7563e0 100644 --- a/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java @@ -593,8 +593,10 @@ public class DeviceEdgeTest extends AbstractEdgeTest { @Test public void testSendDeviceToCloud() throws Exception { - Device deviceMsg = buildDeviceForUplinkMsg("Edge Device 2", "test"); + String deviceName = "Edge Device 2"; + Device deviceMsg = buildDeviceForUplinkMsg(deviceName, "test"); + // create device on edge UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); DeviceUpdateMsg.Builder deviceUpdateMsgBuilder = DeviceUpdateMsg.newBuilder(); deviceUpdateMsgBuilder.setIdMSB(deviceMsg.getUuidId().getMostSignificantBits()); @@ -609,7 +611,25 @@ public class DeviceEdgeTest extends AbstractEdgeTest { Device device = doGet("/api/device/" + deviceMsg.getId().getId(), Device.class); Assert.assertNotNull(device); - Assert.assertEquals("Edge Device 2", device.getName()); + Assert.assertEquals(deviceName, device.getName()); + + // update device on edge + deviceMsg.setName(deviceName + " Updated"); + uplinkMsgBuilder = UplinkMsg.newBuilder(); + deviceUpdateMsgBuilder = DeviceUpdateMsg.newBuilder(); + deviceUpdateMsgBuilder.setIdMSB(deviceMsg.getUuidId().getMostSignificantBits()); + deviceUpdateMsgBuilder.setIdLSB(deviceMsg.getUuidId().getLeastSignificantBits()); + deviceUpdateMsgBuilder.setEntity(JacksonUtil.toString(deviceMsg)); + deviceUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE); + uplinkMsgBuilder.addDeviceUpdateMsg(deviceUpdateMsgBuilder.build()); + + edgeImitator.expectResponsesAmount(1); + edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); + Assert.assertTrue(edgeImitator.waitForResponses()); + + device = doGet("/api/device/" + deviceMsg.getId().getId(), Device.class); + Assert.assertNotNull(device); + Assert.assertEquals(deviceName + " Updated", device.getName()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/edge/RuleChainEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/RuleChainEdgeTest.java index a5fea1694a..7c029e0471 100644 --- a/application/src/test/java/org/thingsboard/server/edge/RuleChainEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/RuleChainEdgeTest.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.edge; -import com.google.protobuf.AbstractMessage; +import com.datastax.oss.driver.api.core.uuid.Uuids; import org.junit.Assert; import org.junit.Test; import org.thingsboard.common.util.JacksonUtil; @@ -29,16 +29,17 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.gen.edge.v1.RuleChainMetadataRequestMsg; import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.edge.v1.UplinkMsg; +import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.UUID; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -74,8 +75,9 @@ public class RuleChainEdgeTest extends AbstractEdgeTest { RuleChainMetaData ruleChainMetaData = JacksonUtil.fromString(ruleChainMetadataUpdateMsg.getEntity(), RuleChainMetaData.class, true); Assert.assertNotNull(ruleChainMetaData); Assert.assertEquals(ruleChainMetaData.getRuleChainId(), savedRuleChain.getId()); - - testRuleChainMetadataRequestMsg(savedRuleChain.getId()); + for (RuleNode ruleNode : ruleChainMetaData.getNodes()) { + Assert.assertEquals(CONFIGURATION_VERSION, ruleNode.getConfigurationVersion()); + } // unassign rule chain from edge edgeImitator.expectMessageAmount(1); @@ -97,60 +99,62 @@ public class RuleChainEdgeTest extends AbstractEdgeTest { } @Test - public void testSendRuleChainMetadataRequestToCloud() throws Exception { - RuleChainId edgeRootRuleChainId = edge.getRootRuleChainId(); - + public void testRuleChainToCloud() throws Exception { + String ruleChainName = "Rule Chain Edge"; + UUID uuid = Uuids.timeBased(); + + // create rule chain on edge + RuleChain edgeRuleChain = new RuleChain(); + edgeRuleChain.setTenantId(tenantId); + edgeRuleChain.setId(new RuleChainId(uuid)); + edgeRuleChain.setName(ruleChainName); UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); - RuleChainMetadataRequestMsg.Builder ruleChainMetadataRequestMsgBuilder = RuleChainMetadataRequestMsg.newBuilder(); - ruleChainMetadataRequestMsgBuilder.setRuleChainIdMSB(edgeRootRuleChainId.getId().getMostSignificantBits()); - ruleChainMetadataRequestMsgBuilder.setRuleChainIdLSB(edgeRootRuleChainId.getId().getLeastSignificantBits()); - testAutoGeneratedCodeByProtobuf(ruleChainMetadataRequestMsgBuilder); - uplinkMsgBuilder.addRuleChainMetadataRequestMsg(ruleChainMetadataRequestMsgBuilder.build()); + RuleChainUpdateMsg.Builder ruleChainUpdateMsgBuilder = RuleChainUpdateMsg.newBuilder(); + ruleChainUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits()); + ruleChainUpdateMsgBuilder.setIdLSB(uuid.getLeastSignificantBits()); + ruleChainUpdateMsgBuilder.setEntity(JacksonUtil.toString(edgeRuleChain)); + ruleChainUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE); + testAutoGeneratedCodeByProtobuf(ruleChainUpdateMsgBuilder); + uplinkMsgBuilder.addRuleChainUpdateMsg(ruleChainUpdateMsgBuilder.build()); testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder); edgeImitator.expectResponsesAmount(1); - edgeImitator.expectMessageAmount(1); edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); + Assert.assertTrue(edgeImitator.waitForResponses()); - Assert.assertTrue(edgeImitator.waitForMessages()); - AbstractMessage latestMessage = edgeImitator.getLatestMessage(); - Assert.assertTrue(latestMessage instanceof RuleChainMetadataUpdateMsg); - RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = (RuleChainMetadataUpdateMsg) latestMessage; - RuleChainMetaData ruleChainMetadataMsg = JacksonUtil.fromString(ruleChainMetadataUpdateMsg.getEntity(), RuleChainMetaData.class, true); - Assert.assertNotNull(ruleChainMetadataMsg); - Assert.assertEquals(edgeRootRuleChainId, ruleChainMetadataMsg.getRuleChainId()); + UplinkResponseMsg latestResponseMsg = edgeImitator.getLatestResponseMsg(); + Assert.assertTrue(latestResponseMsg.getSuccess()); - testAutoGeneratedCodeByProtobuf(ruleChainMetadataUpdateMsg); - } + RuleChain ruleChain = doGet("/api/ruleChain/" + uuid, RuleChain.class); + Assert.assertNotNull(ruleChain); + Assert.assertEquals("Rule Chain Edge", ruleChain.getName()); - private void testRuleChainMetadataRequestMsg(RuleChainId ruleChainId) throws Exception { - RuleChainMetadataRequestMsg.Builder ruleChainMetadataRequestMsgBuilder = RuleChainMetadataRequestMsg.newBuilder() - .setRuleChainIdMSB(ruleChainId.getId().getMostSignificantBits()) - .setRuleChainIdLSB(ruleChainId.getId().getLeastSignificantBits()); - testAutoGeneratedCodeByProtobuf(ruleChainMetadataRequestMsgBuilder); + // update rule chain on edge + edgeRuleChain.setName(ruleChainName + " Updated"); + uplinkMsgBuilder = UplinkMsg.newBuilder(); + ruleChainUpdateMsgBuilder = RuleChainUpdateMsg.newBuilder(); + ruleChainUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits()); + ruleChainUpdateMsgBuilder.setIdLSB(uuid.getLeastSignificantBits()); + ruleChainUpdateMsgBuilder.setEntity(JacksonUtil.toString(edgeRuleChain)); + ruleChainUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE); + testAutoGeneratedCodeByProtobuf(ruleChainUpdateMsgBuilder); + uplinkMsgBuilder.addRuleChainUpdateMsg(ruleChainUpdateMsgBuilder.build()); - UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder() - .addRuleChainMetadataRequestMsg(ruleChainMetadataRequestMsgBuilder.build()); testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder); edgeImitator.expectResponsesAmount(1); - edgeImitator.expectMessageAmount(1); edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); + Assert.assertTrue(edgeImitator.waitForResponses()); - Assert.assertTrue(edgeImitator.waitForMessages()); - AbstractMessage latestMessage = edgeImitator.getLatestMessage(); - Assert.assertTrue(latestMessage instanceof RuleChainMetadataUpdateMsg); - RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = (RuleChainMetadataUpdateMsg) latestMessage; - RuleChainMetaData ruleChainMetadataMsg = JacksonUtil.fromString(ruleChainMetadataUpdateMsg.getEntity(), RuleChainMetaData.class, true); - Assert.assertNotNull(ruleChainMetadataMsg); - Assert.assertEquals(ruleChainId, ruleChainMetadataMsg.getRuleChainId()); + latestResponseMsg = edgeImitator.getLatestResponseMsg(); + Assert.assertTrue(latestResponseMsg.getSuccess()); - for (RuleNode ruleNode : ruleChainMetadataMsg.getNodes()) { - Assert.assertEquals(CONFIGURATION_VERSION, ruleNode.getConfigurationVersion()); - } + ruleChain = doGet("/api/ruleChain/" + uuid, RuleChain.class); + Assert.assertNotNull(ruleChain); + Assert.assertEquals(ruleChainName + " Updated", ruleChain.getName()); } private RuleChainMetaData createRuleChainMetadata(RuleChain ruleChain) { diff --git a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java index 2eb4bde15e..4a303e5253 100644 --- a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java @@ -153,7 +153,7 @@ public class HashPartitionServiceTest { for (int queueIndex = 0; queueIndex < queueCount; queueIndex++) { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, "queue" + queueIndex, tenantId); for (int partition = 0; partition < partitionCount; partition++) { - ServiceInfo serviceInfo = partitionService.resolveByPartitionIdx(services, queueKey, partition, Collections.emptyMap()); + ServiceInfo serviceInfo = partitionService.resolveByPartitionIdx(services, queueKey, partition, Collections.emptyMap()).get(0); String serviceId = serviceInfo.getServiceId(); map.put(serviceId, map.get(serviceId) + 1); } @@ -308,7 +308,7 @@ public class HashPartitionServiceTest { partitionService_common.recalculatePartitions(commonRuleEngine, List.of(dedicatedRuleEngine)); verifyPartitionChangeEvent(event -> { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, TenantId.SYS_TENANT_ID); - return event.getPartitionsMap().get(queueKey).size() == systemQueue.getPartitions(); + return event.getNewPartitions().get(queueKey).size() == systemQueue.getPartitions(); }); Mockito.reset(applicationEventPublisher); @@ -336,14 +336,14 @@ public class HashPartitionServiceTest { // expecting event about no partitions for isolated queue key verifyPartitionChangeEvent(event -> { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId); - return event.getPartitionsMap().get(queueKey).isEmpty(); + return event.getNewPartitions().get(queueKey).isEmpty(); }); partitionService_dedicated.updateQueues(List.of(queueUpdateMsg)); partitionService_dedicated.recalculatePartitions(dedicatedRuleEngine, List.of(commonRuleEngine)); verifyPartitionChangeEvent(event -> { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId); - return event.getPartitionsMap().get(queueKey).size() == isolatedQueue.getPartitions(); + return event.getNewPartitions().get(queueKey).size() == isolatedQueue.getPartitions(); }); @@ -361,7 +361,7 @@ public class HashPartitionServiceTest { partitionService_dedicated.removeQueues(List.of(queueDeleteMsg)); verifyPartitionChangeEvent(event -> { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId); - return event.getPartitionsMap().get(queueKey).isEmpty(); + return event.getNewPartitions().get(queueKey).isEmpty(); }); } @@ -381,12 +381,12 @@ public class HashPartitionServiceTest { Stream.concat(Stream.of(TenantId.SYS_TENANT_ID), Stream.generate(UUID::randomUUID).map(TenantId::new).limit(10)).forEach(tenantId -> { List queues = Stream.generate(() -> RandomStringUtils.randomAlphabetic(10)) .map(queueName -> new QueueKey(ServiceType.TB_RULE_ENGINE, queueName, tenantId)) - .limit(100).collect(Collectors.toList()); + .limit(100).toList(); for (int partition = 0; partition < 10; partition++) { - ServiceInfo expectedAssignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, new QueueKey(ServiceType.TB_RULE_ENGINE, tenantId), partition, Collections.emptyMap()); + ServiceInfo expectedAssignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, new QueueKey(ServiceType.TB_RULE_ENGINE, tenantId), partition, Collections.emptyMap()).get(0); for (QueueKey queueKey : queues) { - ServiceInfo assignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, queueKey, partition, Collections.emptyMap()); + ServiceInfo assignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, queueKey, partition, Collections.emptyMap()).get(0); assertThat(assignedRuleEngine).as(queueKey + "[" + partition + "] should be assigned to " + expectedAssignedRuleEngine.getServiceId()) .isEqualTo(expectedAssignedRuleEngine); } @@ -426,11 +426,15 @@ public class HashPartitionServiceTest { topicService); ReflectionTestUtils.setField(partitionService, "coreTopic", "tb.core"); ReflectionTestUtils.setField(partitionService, "corePartitions", 10); + ReflectionTestUtils.setField(partitionService, "cfEventTopic", "tb_cf_event"); + ReflectionTestUtils.setField(partitionService, "cfStateTopic", "tb_cf_state"); + ReflectionTestUtils.setField(partitionService, "cfPartitions", 10); ReflectionTestUtils.setField(partitionService, "vcTopic", "tb.vc"); ReflectionTestUtils.setField(partitionService, "vcPartitions", 10); ReflectionTestUtils.setField(partitionService, "hashFunctionName", hashFunctionName); ReflectionTestUtils.setField(partitionService, "edgeTopic", "tb.edge"); ReflectionTestUtils.setField(partitionService, "edgePartitions", 10); + ReflectionTestUtils.setField(partitionService, "edqsPartitions", 12); partitionService.init(); partitionService.partitionsInit(); return partitionService; diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java new file mode 100644 index 0000000000..320093e8f4 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -0,0 +1,205 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.tbel.DefaultTbelInvokeService; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; +import org.thingsboard.server.service.cf.CalculatedFieldResult; + +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +@SpringBootTest(classes = DefaultTbelInvokeService.class) +public class ScriptCalculatedFieldStateTest { + + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("5b18e321-3327-4290-b996-d72a65e90382")); + private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); + private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); + + private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new DoubleDataEntry("assetHumidity", 43.0), 122L); + private final TsRollingArgumentEntry deviceTemperatureArgEntry = createRollingArgEntry(); + + private final long ts = System.currentTimeMillis(); + + private ScriptCalculatedFieldState state; + private CalculatedFieldCtx ctx; + + @Autowired + private TbelInvokeService tbelInvokeService; + + @MockBean + private ApiLimitService apiLimitService; + + @BeforeEach + void setUp() { + when(apiLimitService.getLimit(any(), any())).thenReturn(1000L); + ctx = new CalculatedFieldCtx(getCalculatedField(), tbelInvokeService, apiLimitService); + ctx.init(); + state = new ScriptCalculatedFieldState(ctx.getArgNames()); + } + + @Test + void testType() { + assertThat(state.getType()).isEqualTo(CalculatedFieldType.SCRIPT); + } + + @Test + void testUpdateState() { + state.arguments = new HashMap<>(Map.of("assetHumidity", assetHumidityArgEntry)); + + Map newArgs = Map.of("deviceTemperature", deviceTemperatureArgEntry); + boolean stateUpdated = state.updateState(ctx, newArgs); + + assertThat(stateUpdated).isTrue(); + assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( + Map.of( + "assetHumidity", assetHumidityArgEntry, + "deviceTemperature", deviceTemperatureArgEntry + ) + ); + } + + @Test + void testUpdateStateWhenUpdateExistingEntry() { + state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); + + SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(ts, new LongDataEntry("assetHumidity", 41L), 349L); + Map newArgs = Map.of("assetHumidity", newArgEntry); + boolean stateUpdated = state.updateState(ctx, newArgs); + + assertThat(stateUpdated).isTrue(); + assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( + Map.of( + "assetHumidity", newArgEntry, + "deviceTemperature", deviceTemperatureArgEntry + ) + ); + } + + @Test + void testPerformCalculation() throws ExecutionException, InterruptedException { + state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); + + CalculatedFieldResult result = state.performCalculation(ctx).get(); + + assertThat(result).isNotNull(); + Output output = getCalculatedFieldConfig().getOutput(); + assertThat(result.getType()).isEqualTo(output.getType()); + assertThat(result.getScope()).isEqualTo(output.getScope()); + assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("maxDeviceTemperature", 17.0, "assetHumidity", 43.0))); + } + + @Test + void testIsReadyWhenNotAllArgPresent() { + assertThat(state.isReady()).isFalse(); + } + + @Test + void testIsReadyWhenAllArgPresent() { + state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); + + assertThat(state.isReady()).isTrue(); + } + + @Test + void testIsReadyWhenEmptyEntryPresents() { + state.arguments = new HashMap<>(Map.of("deviceTemperature", new TsRollingArgumentEntry(5, 30000L), "assetHumidity", assetHumidityArgEntry)); + + assertThat(state.isReady()).isFalse(); + } + + private TsRollingArgumentEntry createRollingArgEntry() { + TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(5, 30000L); + long ts = System.currentTimeMillis(); + + TreeMap values = new TreeMap<>(); + values.put(ts - 40, 10.0); + values.put(ts - 30, 12.0); + values.put(ts - 20, 17.0); + + argumentEntry.setTsRecords(values); + return argumentEntry; + } + + private CalculatedField getCalculatedField() { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(TENANT_ID); + calculatedField.setEntityId(ASSET_ID); + calculatedField.setType(CalculatedFieldType.SCRIPT); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(getCalculatedFieldConfig()); + calculatedField.setVersion(1L); + return calculatedField; + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig() { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument1 = new Argument(); + argument1.setRefEntityId(DEVICE_ID); + ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("temperature", ArgumentType.TS_ROLLING, null); + argument1.setRefEntityKey(refEntityKey1); + argument1.setLimit(5); + argument1.setTimeWindow(30000L); + + Argument argument2 = new Argument(); + ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("humidity", ArgumentType.TS_LATEST, null); + argument1.setRefEntityKey(refEntityKey2); + + config.setArguments(Map.of("deviceTemperature", argument1, "assetHumidity", argument2)); + + config.setExpression("return {\"maxDeviceTemperature\": deviceTemperature.max(), \"assetHumidity\": assetHumidity}"); + + Output output = new Output(); + output.setType(OutputType.ATTRIBUTES); + output.setScope(AttributeScope.SERVER_SCOPE); + + config.setOutput(output); + + return config; + } + +} \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java new file mode 100644 index 0000000000..f0059e4f6f --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -0,0 +1,250 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; +import org.thingsboard.server.service.cf.CalculatedFieldResult; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class SimpleCalculatedFieldStateTest { + + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("5b18e321-3327-4290-b996-d72a65e90382")); + private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); + private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); + + private final SingleValueArgumentEntry key1ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new LongDataEntry("key1", 11L), 145L); + private final SingleValueArgumentEntry key2ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 6, new LongDataEntry("key2", 15L), 165L); + private final SingleValueArgumentEntry key3ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 3, new LongDataEntry("key3", 23L), 184L); + + private SimpleCalculatedFieldState state; + private CalculatedFieldCtx ctx; + + @Mock + private ApiLimitService apiLimitService; + + @BeforeEach + void setUp() { + when(apiLimitService.getLimit(any(), any())).thenReturn(1000L); + ctx = new CalculatedFieldCtx(getCalculatedField(), null, apiLimitService); + ctx.init(); + state = new SimpleCalculatedFieldState(ctx.getArgNames()); + } + + @Test + void testType() { + assertThat(state.getType()).isEqualTo(CalculatedFieldType.SIMPLE); + } + + @Test + void testUpdateState() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry + )); + + Map newArgs = Map.of("key3", key3ArgEntry); + boolean stateUpdated = state.updateState(ctx, newArgs); + + assertThat(stateUpdated).isTrue(); + assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( + Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry, + "key3", key3ArgEntry + ) + ); + } + + @Test + void testUpdateStateWhenUpdateExistingEntry() { + state.arguments = new HashMap<>(Map.of("key1", key1ArgEntry)); + + SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis(), new LongDataEntry("key1", 18L), 190L); + Map newArgs = Map.of("key1", newArgEntry); + boolean stateUpdated = state.updateState(ctx, newArgs); + + assertThat(stateUpdated).isTrue(); + assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf(Map.of("key1", newArgEntry)); + } + + @Test + void testUpdateStateWhenRollingEntryPassed() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry + )); + + Map newArgs = Map.of("key3", new TsRollingArgumentEntry(10, 30000L)); + assertThatThrownBy(() -> state.updateState(ctx, newArgs)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Rolling argument entry is not supported for simple calculated fields."); + } + + @Test + void testPerformCalculation() throws ExecutionException, InterruptedException { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry, + "key3", key3ArgEntry + )); + + CalculatedFieldResult result = state.performCalculation(ctx).get(); + + assertThat(result).isNotNull(); + Output output = getCalculatedFieldConfig().getOutput(); + assertThat(result.getType()).isEqualTo(output.getType()); + assertThat(result.getScope()).isEqualTo(output.getScope()); + assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 49))); + } + + @Test + void testPerformCalculationWhenPassedNotNumber() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", new SingleValueArgumentEntry(System.currentTimeMillis() - 9, new StringDataEntry("key2", "string"), 124L), + "key3", key3ArgEntry + )); + + assertThatThrownBy(() -> state.performCalculation(ctx)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Argument 'key2' is not a number."); + } + + @Test + void testPerformCalculationWhenDecimalsByDefault() throws ExecutionException, InterruptedException { + state.arguments = new HashMap<>(Map.of( + "key1", new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new DoubleDataEntry("key1", 11.3456), 145L), + "key2", new SingleValueArgumentEntry(System.currentTimeMillis() - 6, new DoubleDataEntry("key2", 15.1), 165L), + "key3", new SingleValueArgumentEntry(System.currentTimeMillis() - 3, new DoubleDataEntry("key3", 23.1), 184L) + )); + + Output output = getCalculatedFieldConfig().getOutput(); + output.setDecimalsByDefault(3); + ctx.setOutput(output); + + CalculatedFieldResult result = state.performCalculation(ctx).get(); + + assertThat(result).isNotNull(); + assertThat(result.getType()).isEqualTo(output.getType()); + assertThat(result.getScope()).isEqualTo(output.getScope()); + assertThat(result.getResult()).isEqualTo(JacksonUtil.valueToTree(Map.of("output", 49.546))); + } + + @Test + void testIsReadyWhenNotAllArgPresent() { + assertThat(state.isReady()).isFalse(); + } + + @Test + void testIsReadyWhenAllArgPresent() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry, + "key3", key3ArgEntry + )); + + assertThat(state.isReady()).isTrue(); + } + + @Test + void testIsReadyWhenEmptyEntryPresents() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry + )); + state.getArguments().put("key3", new SingleValueArgumentEntry()); + + assertThat(state.isReady()).isFalse(); + } + + private CalculatedField getCalculatedField() { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(TENANT_ID); + calculatedField.setEntityId(DEVICE_ID); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(getCalculatedFieldConfig()); + calculatedField.setVersion(1L); + return calculatedField; + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig() { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument1 = new Argument(); + argument1.setRefEntityId(ASSET_ID); + ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("temp1", ArgumentType.TS_LATEST, null); + argument1.setRefEntityKey(refEntityKey1); + + Argument argument2 = new Argument(); + argument2.setRefEntityId(ASSET_ID); + ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("temp2", ArgumentType.ATTRIBUTE, null); + argument2.setRefEntityKey(refEntityKey2); + + Argument argument3 = new Argument(); + argument3.setRefEntityId(ASSET_ID); + ReferencedEntityKey refEntityKey3 = new ReferencedEntityKey("temp3", ArgumentType.TS_LATEST, null); + argument3.setRefEntityKey(refEntityKey3); + + config.setArguments(Map.of("key1", argument1, "key2", argument2, "key3", argument3)); + + config.setExpression("key1 + key2 + key3"); + + Output output = new Output(); + output.setName("output"); + output.setType(OutputType.ATTRIBUTES); + output.setScope(AttributeScope.SERVER_SCOPE); + output.setDecimalsByDefault(0); + + config.setOutput(output); + + return config; + } + +} \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java new file mode 100644 index 0000000000..2c48ed9167 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.kv.LongDataEntry; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class SingleValueArgumentEntryTest { + + private SingleValueArgumentEntry entry; + + private final long ts = System.currentTimeMillis(); + + @BeforeEach + void setUp() { + entry = new SingleValueArgumentEntry(ts, new LongDataEntry("key", 11L), 363L); + } + + @Test + void testArgumentEntryType() { + assertThat(entry.getType()).isEqualTo(ArgumentEntryType.SINGLE_VALUE); + } + + @Test + void testUpdateEntryWhenRollingEntryPassed() { + assertThatThrownBy(() -> entry.updateEntry(new TsRollingArgumentEntry(5, 30000L))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported argument entry type for single value argument entry: " + ArgumentEntryType.TS_ROLLING); + } + + @Test + void testUpdateEntryWithThaSameTs() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts, new LongDataEntry("key", 13L), 363L))).isFalse(); + } + + @Test + void testUpdateEntryWhenNewVersionIsNull() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 16, new LongDataEntry("key", 13L), null))).isTrue(); + assertThat(entry.getValue()).isEqualTo(13L); + assertThat(entry.getVersion()).isNull(); + } + + @Test + void testUpdateEntryWhenNewVersionIsGreaterThanCurrent() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 18L), 369L))).isTrue(); + assertThat(entry.getValue()).isEqualTo(18L); + assertThat(entry.getVersion()).isEqualTo(369L); + } + + @Test + void testUpdateEntryWhenNewVersionIsLessThanCurrent() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 18L), 234L))).isFalse(); + } + + @Test + void testUpdateEntryWhenValueWasNotChanged() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 11L), 364L))).isTrue(); + } +} \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java new file mode 100644 index 0000000000..b1f8063857 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java @@ -0,0 +1,123 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; + +import java.util.Map; +import java.util.TreeMap; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TsRollingArgumentEntryTest { + + private TsRollingArgumentEntry entry; + + private final long ts = System.currentTimeMillis(); + + @BeforeEach + void setUp() { + TreeMap values = new TreeMap<>(); + values.put(ts - 40, 10.0); + values.put(ts - 30, 12.0); + values.put(ts - 20, 17.0); + + entry = new TsRollingArgumentEntry(5, 30000L, values); + } + + @Test + void testArgumentEntryType() { + assertThat(entry.getType()).isEqualTo(ArgumentEntryType.TS_ROLLING); + } + + @Test + void testUpdateEntryWhenSingleValueEntryPassed() { + SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, new DoubleDataEntry("key", 23.0), 123L); + + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords()).hasSize(4); + assertThat(entry.getTsRecords().get(ts - 10)).isEqualTo(23.0); + } + + @Test + void testUpdateEntryWhenRollingEntryPassed() { + TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); + TreeMap values = new TreeMap<>(); + values.put(ts - 10, 7.0); + values.put(ts - 5, 1.0); + newEntry.setTsRecords(values); + + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords()).hasSize(5); + assertThat(entry.getTsRecords()).isEqualTo(Map.of( + ts - 40, 10.0, + ts - 30, 12.0, + ts - 20, 17.0, + ts - 10, 7.0, + ts - 5, 1.0 + )); + } + + @Test + void testUpdateEntryWhenValueIsNotNumber() { + SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, new StringDataEntry("key", "string"), 123L); + + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords().get(ts - 10)).isNaN(); + } + + @Test + void testUpdateEntryWhenOldTelemetry() { + TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); + TreeMap values = new TreeMap<>(); + values.put(ts - 40000, 4.0);// will not be used for calculation + values.put(ts - 45000, 2.0);// will not be used for calculation + values.put(ts - 5, 0.0); + newEntry.setTsRecords(values); + + entry = new TsRollingArgumentEntry(3, 30000L); + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords()).hasSize(1); + assertThat(entry.getTsRecords()).isEqualTo(Map.of( + ts - 5, 0.0 + )); + } + + @Test + void testPerformCalculationWhenArgumentsMoreThanLimit() { + TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); + TreeMap values = new TreeMap<>(); + values.put(ts - 20, 1000.0);// will not be used + values.put(ts - 18, 0.0); + values.put(ts - 16, 0.0); + values.put(ts - 14, 0.0); + newEntry.setTsRecords(values); + + entry = new TsRollingArgumentEntry(3, 30000L); + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords()).hasSize(3); + assertThat(entry.getTsRecords()).isEqualTo(Map.of( + ts - 18, 0.0, + ts - 16, 0.0, + ts - 14, 0.0 + )); + } + +} \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java new file mode 100644 index 0000000000..50c80d08c7 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java @@ -0,0 +1,128 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy; + +import com.google.common.collect.Lists; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.TestPropertySource; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.IdBased; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; +import org.thingsboard.server.common.msg.edqs.EdqsApiService; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.edqs.util.EdqsRocksDb; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.awaitility.Awaitility.await; + +@DaoSqlTest +@TestPropertySource(properties = { + "queue.edqs.sync.enabled=true", + "queue.edqs.api.supported=true", + "queue.edqs.api.auto_enable=true", + "queue.edqs.mode=local" +}) +public class EdqsEntityServiceTest extends EntityServiceTest { + + @Autowired + private EdqsApiService edqsApiService; + + @MockBean + private EdqsRocksDb edqsRocksDb; + + @Before + public void beforeEach() { + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsApiService.isEnabled()); + } + + // sql implementation has a bug with data duplication, edqs implementation returns correct value + @Override + @Test + public void testCountHierarchicalEntitiesByMultiRootQuery() throws InterruptedException { + List buildings = new ArrayList<>(); + List apartments = new ArrayList<>(); + Map> entityNameByTypeMap = new HashMap<>(); + Map childParentRelationMap = new HashMap<>(); + createMultiRootHierarchy(buildings, apartments, entityNameByTypeMap, childParentRelationMap); + + RelationsQueryFilter filter = new RelationsQueryFilter(); + filter.setMultiRoot(true); + filter.setMultiRootEntitiesType(EntityType.ASSET); + filter.setMultiRootEntityIds(buildings.stream().map(IdBased::getId).map(d -> d.getId().toString()).collect(Collectors.toSet())); + filter.setDirection(EntitySearchDirection.FROM); + + EntityCountQuery countQuery = new EntityCountQuery(filter); + countByQueryAndCheck(countQuery, 63); + + filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("AptToHeat", Collections.singletonList(EntityType.DEVICE)))); + countByQueryAndCheck(countQuery, 27); + + filter.setMultiRootEntitiesType(EntityType.ASSET); + filter.setMultiRootEntityIds(apartments.stream().map(IdBased::getId).map(d -> d.getId().toString()).collect(Collectors.toSet())); + filter.setDirection(EntitySearchDirection.TO); + filter.setFilters(Lists.newArrayList( + new RelationEntityTypeFilter("buildingToApt", Collections.singletonList(EntityType.ASSET)), + new RelationEntityTypeFilter("AptToEnergy", Collections.singletonList(EntityType.DEVICE)))); + countByQueryAndCheck(countQuery, 3); + + deviceService.deleteDevicesByTenantId(tenantId); + assetService.deleteAssetsByTenantId(tenantId); + } + + @Override + protected PageData findByQueryAndCheck(CustomerId customerId, EntityDataQuery query, long expectedResultSize) { + return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findByQuery(customerId, query), + result -> result.getTotalElements() == expectedResultSize); + } + + @Override + protected List findByQueryAndCheckTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List expectedTelemetries) { + return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findEntitiesTelemetry(query, entityKeyType, key, expectedTelemetries), + loadedEntities -> loadedEntities.stream().map(entityData -> entityData.getLatest().get(entityKeyType).get(key).getValue()).toList().containsAll(expectedTelemetries)); + } + + @Override + protected long countByQueryAndCheck(EntityCountQuery countQuery, int expectedResult) { + return countByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), countQuery, expectedResult); + } + + @Override + protected long countByQueryAndCheck(CustomerId customerId, EntityCountQuery query, int expectedResult) { + return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> countByQuery(customerId, query), + result -> result == expectedResult); + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java similarity index 80% rename from dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java rename to application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index 0a0b5478e7..18687cedba 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -13,22 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.service; +package org.thingsboard.server.service.entitiy; import com.google.common.collect.Lists; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.ResultSetExtractor; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; @@ -37,6 +40,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.IdBased; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -46,6 +50,8 @@ import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; +import org.thingsboard.server.common.data.objects.TelemetryEntityView; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.ApiUsageStateFilter; import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; @@ -63,6 +69,7 @@ import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.EntityListFilter; import org.thingsboard.server.common.data.query.EntityNameFilter; +import org.thingsboard.server.common.data.query.EntityViewTypeFilter; import org.thingsboard.server.common.data.query.FilterPredicateValue; import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.NumericFilterPredicate; @@ -75,17 +82,22 @@ import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardDao; +import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.entityview.EntityViewDao; +import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.sql.relation.RelationRepository; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; @@ -105,13 +117,12 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.assertEquals; import static org.thingsboard.server.common.data.query.EntityKeyType.ATTRIBUTE; import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIELD; @Slf4j @DaoSqlTest -public class EntityServiceTest extends AbstractServiceTest { +public class EntityServiceTest extends AbstractControllerTest { static final int ENTITY_COUNT = 5; public static final String TEST_CUSTOMER_NAME = "Test"; @@ -119,6 +130,12 @@ public class EntityServiceTest extends AbstractServiceTest { @Autowired AssetService assetService; @Autowired + AssetProfileService assetProfileService; + @Autowired + DashboardService dashboardService; + @Autowired + EntityViewService entityViewService; + @Autowired UserService userService; @Autowired AttributesService attributesService; @@ -157,7 +174,7 @@ public class EntityServiceTest extends AbstractServiceTest { } @Test - public void testCountEntitiesByQuery() throws InterruptedException { + public void testCountEntitiesByQuery() { List devices = new ArrayList<>(); for (int i = 0; i < 97; i++) { Device device = new Device(); @@ -173,33 +190,26 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setDeviceNameFilter(""); EntityCountQuery countQuery = new EntityCountQuery(filter); - - long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(97, count); + countByQueryAndCheck(countQuery, 97); filter.setDeviceTypes(List.of("unknown")); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); filter.setDeviceTypes(List.of("default")); filter.setDeviceNameFilter("Device1"); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(11, count); + countByQueryAndCheck(countQuery, 11); EntityListFilter entityListFilter = new EntityListFilter(); entityListFilter.setEntityType(EntityType.DEVICE); entityListFilter.setEntityList(devices.stream().map(Device::getId).map(DeviceId::toString).collect(Collectors.toList())); countQuery = new EntityCountQuery(entityListFilter); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(97, count); + countByQueryAndCheck(countQuery, 97); deviceService.deleteDevicesByTenantId(tenantId); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); } - @Test public void testCountHierarchicalEntitiesByQuery() throws InterruptedException { List assets = new ArrayList<>(); @@ -211,19 +221,15 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setDirection(EntitySearchDirection.FROM); EntityCountQuery countQuery = new EntityCountQuery(filter); - - long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(31, count); //due to the loop relations in hierarchy, the TenantId included in total count (1*Tenant + 5*Asset + 5*5*Devices = 31) + countByQueryAndCheck(countQuery, 31); //due to the loop relations in hierarchy, the TenantId included in total count (1*Tenant + 5*Asset + 5*5*Devices = 31) filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("Contains", Collections.singletonList(EntityType.DEVICE)))); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(25, count); + countByQueryAndCheck(countQuery, 25); filter.setRootEntity(devices.get(0).getId()); filter.setDirection(EntitySearchDirection.TO); filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("Manages", Collections.singletonList(EntityType.TENANT)))); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(1, count); + countByQueryAndCheck(countQuery, 1); DeviceSearchQueryFilter filter2 = new DeviceSearchQueryFilter(); filter2.setRootEntity(tenantId); @@ -231,18 +237,14 @@ public class EntityServiceTest extends AbstractServiceTest { filter2.setRelationType("Contains"); countQuery = new EntityCountQuery(filter2); - - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(25, count); + countByQueryAndCheck(countQuery, 25); filter2.setDeviceTypes(Arrays.asList("default0", "default1")); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(10, count); + countByQueryAndCheck(countQuery, 10); filter2.setRootEntity(devices.get(0).getId()); filter2.setDirection(EntitySearchDirection.TO); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); AssetSearchQueryFilter filter3 = new AssetSearchQueryFilter(); filter3.setRootEntity(tenantId); @@ -250,18 +252,14 @@ public class EntityServiceTest extends AbstractServiceTest { filter3.setRelationType("Manages"); countQuery = new EntityCountQuery(filter3); - - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(5, count); + countByQueryAndCheck(countQuery, 5); filter3.setAssetTypes(Arrays.asList("type0", "type1")); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(2, count); + countByQueryAndCheck(countQuery, 2); filter3.setRootEntity(devices.get(0).getId()); filter3.setDirection(EntitySearchDirection.TO); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); } @Test @@ -278,11 +276,12 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - PageData entityDataByQuery = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData entityDataByQuery = findByQueryAndCheck(query, 5); List data = entityDataByQuery.getData(); Assert.assertEquals(data.size(), 5); data.forEach(entityData -> Assert.assertNotNull(entityData.getLatest().get(EntityKeyType.ENTITY_FIELD).get("phone"))); + countByQueryAndCheck(query, 5); } private void createTestUserRelations(TenantId tenantId, List users) { @@ -312,30 +311,24 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setEdgeNameFilter(""); EntityCountQuery countQuery = new EntityCountQuery(filter); - - long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(97, count); + countByQueryAndCheck(countQuery, 97); filter.setEdgeTypes(List.of("unknown")); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); filter.setEdgeTypes(List.of("default")); filter.setEdgeNameFilter("Edge1"); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(11, count); + countByQueryAndCheck(countQuery, 11); EntityListFilter entityListFilter = new EntityListFilter(); entityListFilter.setEntityType(EntityType.EDGE); entityListFilter.setEntityList(edges.stream().map(Edge::getId).map(EdgeId::toString).collect(Collectors.toList())); countQuery = new EntityCountQuery(entityListFilter); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(97, count); + countByQueryAndCheck(countQuery, 97); edgeService.deleteEdgesByTenantId(tenantId); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); } @Test @@ -360,13 +353,10 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setRelationType("Manages"); EntityCountQuery countQuery = new EntityCountQuery(filter); - - long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(5, count); + countByQueryAndCheck(countQuery, 5); filter.setEdgeTypes(Arrays.asList("type0", "type1")); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(2, count); + countByQueryAndCheck(countQuery, 2); } private Edge createEdge(int i, String type) { @@ -423,19 +413,10 @@ public class EntityServiceTest extends AbstractServiceTest { List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); - EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(25, loadedEntities.size()); - List loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); + findByQueryAndCheckTelemetry(query, EntityKeyType.ATTRIBUTE, "temperature", deviceTemperatures); pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = new KeyFilter(); @@ -446,23 +427,10 @@ public class EntityServiceTest extends AbstractServiceTest { highTemperatureFilter.setPredicate(predicate); List keyFilters = Collections.singletonList(highTemperatureFilter); - query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - - loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); - - List loadedHighTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); List deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); + query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + findByQueryAndCheckTelemetry(query, EntityKeyType.ATTRIBUTE, "temperature", deviceHighTemperatures); deviceService.deleteDevicesByTenantId(tenantId); } @@ -482,13 +450,10 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setDirection(EntitySearchDirection.FROM); EntityCountQuery countQuery = new EntityCountQuery(filter); - - long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(63, count); + countByQueryAndCheck(countQuery, 63); filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("AptToHeat", Collections.singletonList(EntityType.DEVICE)))); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(27, count); + countByQueryAndCheck(countQuery, 27); filter.setMultiRootEntitiesType(EntityType.ASSET); filter.setMultiRootEntityIds(apartments.stream().map(IdBased::getId).map(d -> d.getId().toString()).collect(Collectors.toSet())); @@ -496,13 +461,10 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setFilters(Lists.newArrayList( new RelationEntityTypeFilter("buildingToApt", Collections.singletonList(EntityType.ASSET)), new RelationEntityTypeFilter("AptToEnergy", Collections.singletonList(EntityType.DEVICE)))); - - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(9, count); + countByQueryAndCheck(countQuery, 9); deviceService.deleteDevicesByTenantId(tenantId); assetService.deleteAssetsByTenantId(tenantId); - } @Test @@ -538,15 +500,6 @@ public class EntityServiceTest extends AbstractServiceTest { onlineStatusFilter.setPredicate(predicate); List keyFilters = Collections.singletonList(onlineStatusFilter); - EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - long expectedEntitiesCnt = entityNameByTypeMap.entrySet() .stream() .filter(e -> !e.getKey().equals("building")) @@ -554,6 +507,14 @@ public class EntityServiceTest extends AbstractServiceTest { .map(Map.Entry::getValue) .filter(e -> StringUtils.endsWith(e, "_1")) .count(); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + PageData data = findByQueryAndCheck(query, expectedEntitiesCnt); + List loadedEntities = new ArrayList<>(data.getData()); + while (data.hasNext()) { + query = query.next(); + data = findByQuery(query); + loadedEntities.addAll(data.getData()); + } Assert.assertEquals(expectedEntitiesCnt, loadedEntities.size()); Map actualRelations = new HashMap<>(); @@ -602,20 +563,12 @@ public class EntityServiceTest extends AbstractServiceTest { List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); + List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(25, loadedEntities.size()); + List loadedEntities = findByQueryAndCheckTelemetry(query, EntityKeyType.ATTRIBUTE, "temperature", deviceTemperatures); + loadedEntities.forEach(entity -> Assert.assertTrue(devices.stream().map(Device::getId).collect(Collectors.toSet()).contains(entity.getEntityId()))); - List loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); - List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - Assert.assertEquals(deviceTemperatures, loadedTemperatures); pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = new KeyFilter(); @@ -628,21 +581,8 @@ public class EntityServiceTest extends AbstractServiceTest { query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - - loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); - - List loadedHighTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); List deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.ATTRIBUTE, "temperature", deviceHighTemperatures); deviceService.deleteDevicesByTenantId(tenantId); } @@ -676,18 +616,9 @@ public class EntityServiceTest extends AbstractServiceTest { List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "consumption")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(5, loadedEntities.size()); - List loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("consumption").getValue()).collect(Collectors.toList()); + List deviceTemperatures = consumptions.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.ATTRIBUTE, "consumption", deviceTemperatures); pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = new KeyFilter(); @@ -700,21 +631,8 @@ public class EntityServiceTest extends AbstractServiceTest { query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - - loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(highConsumptions.size(), loadedEntities.size()); - - List loadedHighTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.ATTRIBUTE).get("consumption").getValue()).collect(Collectors.toList()); List deviceHighTemperatures = highConsumptions.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.ATTRIBUTE, "consumption", deviceHighTemperatures); deviceService.deleteDevicesByTenantId(tenantId); } @@ -896,9 +814,7 @@ public class EntityServiceTest extends AbstractServiceTest { List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - - Assert.assertEquals(97, data.getTotalElements()); + PageData data = findByQueryAndCheck(query, 97); Assert.assertEquals(10, data.getTotalPages()); Assert.assertTrue(data.hasNext()); Assert.assertEquals(10, data.getData().size()); @@ -906,7 +822,7 @@ public class EntityServiceTest extends AbstractServiceTest { List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(97, loadedEntities.size()); @@ -931,7 +847,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(10, 0, "device1", sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); Assert.assertEquals(11, data.getTotalElements()); Assert.assertEquals("Device19", data.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); @@ -945,11 +861,12 @@ public class EntityServiceTest extends AbstractServiceTest { devices.get(1).setLabel(null); devices.forEach(deviceService::saveDevice); + // FIXME (for Dasha, plz investigate): + // this and other tests below submit an empty value to a KEY FILTER, this is not "search text". + // why are we supposed to ignore it and return all devices? maybe it's a bug? String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.EQUAL, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -961,9 +878,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = devices.get(2).getLabel(); EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.NOT_EQUAL, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size() - 1, result.getTotalElements()); + findByQueryAndCheck(query, devices.size() - 1); } @Test @@ -976,8 +891,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.NOT_EQUAL, searchQuery); - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -989,9 +903,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.STARTS_WITH, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -1003,9 +915,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.ENDS_WITH, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -1017,9 +927,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.CONTAINS, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -1031,9 +939,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = "label-"; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.NOT_CONTAINS, searchQuery); - - PageData result = searchEntities(query); - assertEquals(2, result.getTotalElements()); + findByQueryAndCheck(query, 2); } @Test @@ -1045,9 +951,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.NOT_CONTAINS, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -1071,34 +975,27 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setEntityNameFilter("Device%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setEntityNameFilter("%Device%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setEntityNameFilter("%Device"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test public void testFindEntityDataByQuery_filter_entity_name_ends_with() { List devices = new ArrayList<>(); + String suffixes = RandomStringUtils.randomAlphanumeric(5); for (int i = 0; i < 10; i++) { Device device = new Device(); device.setTenantId(tenantId); - device.setName("Device " + i + " test"); + device.setName("Device " + i + suffixes); device.setType("default"); devices.add(device); } @@ -1107,29 +1004,21 @@ public class EntityServiceTest extends AbstractServiceTest { EntityNameFilter deviceTypeFilter = new EntityNameFilter(); deviceTypeFilter.setEntityType(EntityType.DEVICE); - deviceTypeFilter.setEntityNameFilter("%test"); + deviceTypeFilter.setEntityNameFilter("%" + suffixes); EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); + findByQueryAndCheck(query, devices.size()); - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + deviceTypeFilter.setEntityNameFilter("%" + suffixes + "%"); + findByQueryAndCheck(query, devices.size()); - deviceTypeFilter.setEntityNameFilter("%test%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); - - deviceTypeFilter.setEntityNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + deviceTypeFilter.setEntityNameFilter(suffixes + "%"); + findByQueryAndCheck(query, 0); - deviceTypeFilter.setEntityNameFilter("test"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + deviceTypeFilter.setEntityNameFilter(suffixes); + findByQueryAndCheck(query, 0); } @Test @@ -1153,19 +1042,13 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setEntityNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); deviceTypeFilter.setEntityNameFilter("%test"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1189,24 +1072,16 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("Device%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("%Device%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("%Device"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1232,31 +1107,12 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataQuery query = new EntityDataQuery(singleEntityFilter, pageLink, entityFields, null, null); - PageData result = searchEntities(query); - assertEquals(1, result.getTotalElements()); + PageData result = findByQueryAndCheck(query, 1); String deviceName = result.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); assertThat(deviceName).isEqualTo(devices.get(0).getName()); } - @Test - public void testFindEntitiesByApiUsageStateFilter() { - apiUsageStateService.createDefaultApiUsageState(tenantId, customerId); - ApiUsageStateFilter apiUsageStateFilter = new ApiUsageStateFilter(); - apiUsageStateFilter.setCustomerId(customerId); - - List entityFields = List.of( - new EntityKey(EntityKeyType.ENTITY_FIELD, "name") - ); - - EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); - EntityDataQuery query = new EntityDataQuery(apiUsageStateFilter, pageLink, entityFields, null, null); - PageData result = searchEntities(query); - assertEquals(1, result.getTotalElements()); - String name = result.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); - assertThat(name).isEqualTo(TEST_CUSTOMER_NAME); - } - @Test public void testFindEntitiesByRelationEntityTypeFilter() { Customer customer = new Customer(); @@ -1312,11 +1168,8 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setRootEntity(asset.getId()); EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); - PageData relationsResult = entityService.findEntityDataByQuery(tenantId, customer.getId(), query); - long relationsResultCnt = entityService.countEntitiesByQuery(tenantId, customer.getId(), query); - - Assert.assertEquals(relationsCnt, relationsResult.getData().size()); - Assert.assertEquals(relationsCnt, relationsResultCnt); + findByQueryAndCheck(customer.getId(), query, relationsCnt); + countByQueryAndCheck(customer.getId(), query, relationsCnt); } } @@ -1341,24 +1194,16 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("%test%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); deviceTypeFilter.setDeviceNameFilter("test"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1382,19 +1227,13 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); deviceTypeFilter.setDeviceNameFilter("%test"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1418,24 +1257,16 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(assetTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("Asset%"); - - result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("%Asset%"); - - result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("%Asset"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1459,24 +1290,16 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(assetTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("%test%"); - - result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); assetTypeFilter.setAssetNameFilter("test"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1488,6 +1311,7 @@ public class EntityServiceTest extends AbstractServiceTest { asset.setTenantId(tenantId); asset.setName("Asset test" + i); asset.setType("default"); + asset.setAssetProfileId(assetProfileService.findDefaultAssetProfile(tenantId).getId()); assets.add(asset); } @@ -1500,25 +1324,105 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(assetTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); assetTypeFilter.setAssetNameFilter("%test"); + findByQueryAndCheck(query, 0); + } + + @Test + public void testFindEntitiesBySingleEntityFilter_customer() { + List customerDevices = new ArrayList<>(); + List tenantDevices = new ArrayList<>(); + + for (int i = 0; i < 3; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setCustomerId(customerId); + device.setName("Device test" + i); + device.setType("default"); + Device saved = deviceService.saveDevice(device); + customerDevices.add(saved); + } + + for (int i = 0; i < 3; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Tenant test device" + i); + device.setType("default"); + tenantDevices.add(deviceService.saveDevice(device)); + } + + SingleEntityFilter singleEntityFilter = new SingleEntityFilter(); + singleEntityFilter.setSingleEntity(customerDevices.get(0).getId()); + List entityFields = List.of( + new EntityKey(EntityKeyType.ENTITY_FIELD, "name") + ); + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + EntityDataQuery query = new EntityDataQuery(singleEntityFilter, pageLink, entityFields, null, null); - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + PageData result = findByQueryAndCheck(query, 1); + String deviceName = result.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(deviceName).isEqualTo(customerDevices.get(0).getName()); + + // find by customer user with generic permission + PageData customerResults = findByQueryAndCheck(customerId, query, 1); + + String cutomerDeviceName = customerResults.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(cutomerDeviceName).isEqualTo(customerDevices.get(0).getName()); + + // try to find tenant device by customer user + SingleEntityFilter tenantDeviceFilter = new SingleEntityFilter(); + tenantDeviceFilter.setSingleEntity(tenantDevices.get(0).getId()); + EntityDataQuery customerQuery2 = new EntityDataQuery(tenantDeviceFilter, pageLink, entityFields, null, null); + findByQueryAndCheck(customerId, customerQuery2, 0); } - private PageData searchEntities(EntityDataQuery query) { - return entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + private List getResultDeviceIds(PageData result) { + return result.getData().stream().map(entityData -> (DeviceId) entityData.getEntityId()).collect(Collectors.toList()); } + private Device createDevice(CustomerId customerId) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setCustomerId(customerId); + device.setName("Device test " + RandomStringUtils.randomAlphabetic(5)); + device.setType("default"); + return device; + } + + @Test + public void testFindEntitiesByApiUsageStateFilter() { + ApiUsageStateFilter apiUsageStateFilter = new ApiUsageStateFilter(); + + List entityFields = List.of( + new EntityKey(EntityKeyType.ENTITY_FIELD, "name") + ); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + EntityDataQuery query = new EntityDataQuery(apiUsageStateFilter, pageLink, entityFields, null, null); + PageData result = findByQueryAndCheck(query, 1); + String name = result.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(name).isEqualTo(TEST_TENANT_NAME); + + // find by customer user with generic permissions + apiUsageStateService.createDefaultApiUsageState(tenantId, customerId); + PageData customerResult = findByQueryAndCheck(customerId, query, 1); + + String customerResultName = customerResult.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(customerResultName).isEqualTo(TEST_CUSTOMER_NAME); + + // find by tenant user with customerId filter + apiUsageStateFilter.setCustomerId(customerId); + PageData tenantResult = findByQueryAndCheck(query, 1); + String tenantResultName = tenantResult.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(tenantResultName).isEqualTo(TEST_CUSTOMER_NAME); + } + + private EntityDataQuery createDeviceSearchQuery(String deviceField, StringOperation operation, String searchQuery) { DeviceTypeFilter deviceTypeFilter = new DeviceTypeFilter(); deviceTypeFilter.setDeviceTypes(List.of("default")); @@ -1597,45 +1501,15 @@ public class EntityServiceTest extends AbstractServiceTest { for (EntityKeyType currentAttributeKeyType : attributesEntityTypes) { List latestValues = Collections.singletonList(new EntityKey(currentAttributeKeyType, "temperature")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(67, loadedEntities.size()); - List loadedTemperatures = new ArrayList<>(); - for (Device device : devices) { - loadedTemperatures.add(loadedEntities.stream().filter(entityData -> entityData.getEntityId().equals(device.getId())).findFirst().orElse(null) - .getLatest().get(currentAttributeKeyType).get("temperature").getValue()); - } - List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).toList(); + findByQueryAndCheckTelemetry(query, currentAttributeKeyType, "temperature", deviceTemperatures); pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = createNumericKeyFilter("temperature", currentAttributeKeyType, NumericFilterPredicate.NumericOperation.GREATER, 45); List keyFiltersHighTemperature = Collections.singletonList(highTemperatureFilter); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersHighTemperature); - - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - - loadedEntities = new ArrayList<>(data.getData()); - - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); - - List loadedHighTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(currentAttributeKeyType).get("temperature").getValue()).collect(Collectors.toList()); - List deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); - + findByQueryAndCheckTelemetry(query, currentAttributeKeyType, "temperature", highTemperatures.stream().map(Object::toString).toList()); } deviceService.deleteDevicesByTenantId(tenantId); } @@ -1715,89 +1589,48 @@ public class EntityServiceTest extends AbstractServiceTest { List keyFiltersNotEqualTemperature = Collections.singletonList(notEqualTemperatureFilter); //Greater Operation + List deviceTemperatures = greaterTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, sortOrder); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersGreaterTemperature); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - List loadedEntities = getLoadedEntities(data, query); - Assert.assertEquals(greaterTemperatures.size(), loadedEntities.size()); - List loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); - List deviceTemperatures = greaterTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.CLIENT_ATTRIBUTE, "temperature", deviceTemperatures); //Greater or equal Operation + deviceTemperatures = greaterOrEqualTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersGreaterOrEqualTemperature); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities = getLoadedEntities(data, query); - Assert.assertEquals(greaterOrEqualTemperatures.size(), loadedEntities.size()); - - loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); - deviceTemperatures = greaterOrEqualTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.CLIENT_ATTRIBUTE, "temperature", deviceTemperatures); //Less Operation + deviceTemperatures = lessTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersLessTemperature); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities = getLoadedEntities(data, query); - Assert.assertEquals(lessTemperatures.size(), loadedEntities.size()); - - loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); - deviceTemperatures = lessTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.CLIENT_ATTRIBUTE, "temperature", deviceTemperatures); //Less or equal Operation + deviceTemperatures = lessOrEqualTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersLessOrEqualTemperature); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities = getLoadedEntities(data, query); - Assert.assertEquals(lessOrEqualTemperatures.size(), loadedEntities.size()); - - loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); - deviceTemperatures = lessOrEqualTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.CLIENT_ATTRIBUTE, "temperature", deviceTemperatures); //Equal Operation + deviceTemperatures = equalTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersEqualTemperature); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities = getLoadedEntities(data, query); - Assert.assertEquals(equalTemperatures.size(), loadedEntities.size()); - - loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); - deviceTemperatures = equalTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + findByQueryAndCheckTelemetry(query, EntityKeyType.CLIENT_ATTRIBUTE, "temperature", deviceTemperatures); //Not equal Operation + deviceTemperatures = notEqualTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersNotEqualTemperature); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities = getLoadedEntities(data, query); - Assert.assertEquals(notEqualTemperatures.size(), loadedEntities.size()); - - loadedTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("temperature").getValue()).collect(Collectors.toList()); - deviceTemperatures = notEqualTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - Assert.assertEquals(deviceTemperatures, loadedTemperatures); - + findByQueryAndCheckTelemetry(query, EntityKeyType.CLIENT_ATTRIBUTE, "temperature", deviceTemperatures); deviceService.deleteDevicesByTenantId(tenantId); } @@ -1824,7 +1657,7 @@ public class EntityServiceTest extends AbstractServiceTest { } } - List> timeseriesFutures = new ArrayList<>(); + List> timeseriesFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); timeseriesFutures.add(saveLongTimeseries(device.getId(), "temperature", temperatures.get(i))); @@ -1842,24 +1675,10 @@ public class EntityServiceTest extends AbstractServiceTest { List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); - EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - - List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(67, loadedEntities.size()); - List loadedTemperatures = new ArrayList<>(); - for (Device device : devices) { - loadedTemperatures.add(loadedEntities.stream().filter(entityData -> entityData.getEntityId().equals(device.getId())).findFirst().orElse(null) - .getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()); - } List deviceTemperatures = temperatures.stream().map(aDouble -> Double.toString(aDouble)).collect(Collectors.toList()); - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); + findByQueryAndCheckTelemetry(query, EntityKeyType.TIME_SERIES, "temperature", deviceTemperatures); pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = new KeyFilter(); @@ -1870,23 +1689,10 @@ public class EntityServiceTest extends AbstractServiceTest { highTemperatureFilter.setPredicate(predicate); List keyFilters = Collections.singletonList(highTemperatureFilter); - query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - - loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); - - List loadedHighTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()).collect(Collectors.toList()); List deviceHighTemperatures = highTemperatures.stream().map(aDouble -> Double.toString(aDouble)).collect(Collectors.toList()); - Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); + query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + findByQueryAndCheckTelemetry(query, EntityKeyType.TIME_SERIES, "temperature", deviceHighTemperatures); deviceService.deleteDevicesByTenantId(tenantId); } @@ -1994,7 +1800,7 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, sortOrder); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersEqualString); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, equalStrings.size()); List loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(equalStrings.size(), loadedEntities.size()); @@ -2007,7 +1813,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersNotEqualString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, notEqualStrings.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(notEqualStrings.size(), loadedEntities.size()); @@ -2020,7 +1826,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersStartsWithString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, startsWithStrings.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(startsWithStrings.size(), loadedEntities.size()); @@ -2033,7 +1839,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersEndsWithString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, endsWithStrings.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(endsWithStrings.size(), loadedEntities.size()); @@ -2046,7 +1852,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersContainsString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, containsStrings.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(containsStrings.size(), loadedEntities.size()); @@ -2059,7 +1865,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersNotContainsString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, notContainsStrings.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(notContainsStrings.size(), loadedEntities.size()); @@ -2072,7 +1878,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, deviceTypeFilters); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2117,7 +1923,7 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, sortOrder); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersEqualString); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, devices.size()); List loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2132,7 +1938,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersNotEqualString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2145,7 +1951,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersStartsWithString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2158,7 +1964,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersEndsWithString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2171,7 +1977,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersContainsString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2184,7 +1990,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersNotContainsString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2232,7 +2038,7 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, sortOrder); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, deviceTypeFilters); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, devices.size()); List loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2240,7 +2046,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, createdTimeFilters); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2248,7 +2054,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, null); query = new EntityDataQuery(filter, pageLink, entityFields, null, nameFilters); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2296,12 +2102,12 @@ public class EntityServiceTest extends AbstractServiceTest { // query with textSearch - optimization is not performing EntityDataPageLink originalPageLink = new EntityDataPageLink(pageSize, 0, "Device", sortOrder); EntityDataQuery originalQuery = new EntityDataQuery(filter, originalPageLink, entityFields, null, deviceTypeFilters); - PageData originalData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), originalQuery); + PageData originalData = findByQueryAndCheck(originalQuery, expectedDevicesSize); // query without textSearch - optimization is performing EntityDataPageLink optimizedPageLink = new EntityDataPageLink(pageSize, 0, null, sortOrder); EntityDataQuery optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, deviceTypeFilters); - PageData optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery); + PageData optimizedData = findByQueryAndCheck(optimizedQuery, expectedDevicesSize); List loadedEntities = getLoadedEntities(optimizedData, optimizedQuery); Assert.assertEquals(expectedDevicesSize, loadedEntities.size()); loadedEntities = getLoadedEntities(originalData, originalQuery); @@ -2325,12 +2131,12 @@ public class EntityServiceTest extends AbstractServiceTest { // query with textSearch - optimization is not performing originalPageLink = new EntityDataPageLink(pageSize, 0, "Device", sortOrder); originalQuery = new EntityDataQuery(filter, originalPageLink, entityFields, null, attributeFilters); - originalData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), originalQuery); + originalData = findByQuery(originalQuery); // query without textSearch - optimization is performing optimizedPageLink = new EntityDataPageLink(pageSize, 0, null, sortOrder); optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, attributeFilters); - optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery); + optimizedData = findByQuery(optimizedQuery); loadedEntities = getLoadedEntities(optimizedData, optimizedQuery); Assert.assertEquals(expectedDevicesSize, loadedEntities.size()); loadedEntities = getLoadedEntities(originalData, originalQuery); @@ -2354,12 +2160,12 @@ public class EntityServiceTest extends AbstractServiceTest { // query with textSearch - optimization is not performing originalPageLink = new EntityDataPageLink(pageSize, 0, "Device", sortOrder); originalQuery = new EntityDataQuery(filter, originalPageLink, entityFields, null, nameFilters); - originalData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), originalQuery); + originalData = findByQuery(originalQuery); // query without textSearch - optimization is performing optimizedPageLink = new EntityDataPageLink(pageSize, 0, null, sortOrder); optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, nameFilters); - optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery); + optimizedData = findByQuery(optimizedQuery); loadedEntities = getLoadedEntities(optimizedData, optimizedQuery); Assert.assertEquals(expectedDevicesSize, loadedEntities.size()); loadedEntities = getLoadedEntities(originalData, originalQuery); @@ -2387,10 +2193,9 @@ public class EntityServiceTest extends AbstractServiceTest { private List getLoadedEntities(PageData data, EntityDataQuery query) { List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } return loadedEntities; @@ -2421,25 +2226,25 @@ public class EntityServiceTest extends AbstractServiceTest { private ListenableFuture> saveLongAttribute(EntityId entityId, String key, long value, AttributeScope scope) { KvEntry attrValue = new LongDataEntry(key, value); AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L); - return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr)); + return attributesService.save(tenantId, entityId, scope, Collections.singletonList(attr)); } private ListenableFuture> saveStringAttribute(EntityId entityId, String key, String value, AttributeScope scope) { KvEntry attrValue = new StringDataEntry(key, value); AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L); - return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr)); + return attributesService.save(tenantId, entityId, scope, Collections.singletonList(attr)); } - private ListenableFuture saveLongTimeseries(EntityId entityId, String key, Double value) { + private ListenableFuture saveLongTimeseries(EntityId entityId, String key, Double value) { TsKvEntity tsKv = new TsKvEntity(); tsKv.setStrKey(key); tsKv.setDoubleValue(value); KvEntry telemetryValue = new DoubleDataEntry(key, value); BasicTsKvEntry timeseries = new BasicTsKvEntry(42L, telemetryValue); - return timeseriesService.save(SYSTEM_TENANT_ID, entityId, timeseries); + return timeseriesService.save(tenantId, entityId, timeseries); } - private void createMultiRootHierarchy(List buildings, List apartments, + protected void createMultiRootHierarchy(List buildings, List apartments, Map> entityNameByTypeMap, Map childParentRelationMap) throws InterruptedException { for (int k = 0; k < 3; k++) { @@ -2510,4 +2315,93 @@ public class EntityServiceTest extends AbstractServiceTest { } } } + + @Test + public void testFindEntitiesWithEntityViewFilter() { + EntityView entityView = new EntityView(); + entityView.setTenantId(tenantId); + entityView.setCustomerId(customerId); + entityView.setName("test"); + entityView.setType("default"); + entityView.setEntityId(new DeviceId(UUID.randomUUID())); + entityView.setKeys(new TelemetryEntityView(List.of("test"), null)); + entityView.setStartTimeMs(124); + entityView.setEndTimeMs(256); + entityView.setExternalId(new EntityViewId(UUID.randomUUID())); + entityView.setAdditionalInfo(JacksonUtil.newObjectNode().put("test", "test")); + entityView = entityViewService.saveEntityView(entityView); + + EntityViewTypeFilter entityViewTypeFilter = new EntityViewTypeFilter(); + entityViewTypeFilter.setEntityViewNameFilter("test"); + entityViewTypeFilter.setEntityViewTypes(List.of("non-existing", "default")); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List entityFields = List.of( + new EntityKey(EntityKeyType.ENTITY_FIELD, "name") + ); + EntityDataQuery query = new EntityDataQuery(entityViewTypeFilter, pageLink, entityFields, Collections.emptyList(), null); + + PageData relationsResult = findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), query, 1); + assertThat(relationsResult.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()).isEqualTo(entityView.getName()); + + // find with non existing name + entityViewTypeFilter.setEntityViewNameFilter("non-existing"); + findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), query, 0); + + // find with non existing type + entityViewTypeFilter.setEntityViewNameFilter(null); + entityViewTypeFilter.setEntityViewTypes(Collections.singletonList("non-existing")); + + findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), query, 0); + } + + protected PageData findByQuery(EntityDataQuery query) { + return findByQuery(new CustomerId(CustomerId.NULL_UUID), query); + } + + protected PageData findByQuery(CustomerId customerId, EntityDataQuery query) { + return entityService.findEntityDataByQuery(tenantId, customerId, query); + } + + protected PageData findByQueryAndCheck(EntityDataQuery query, long expectedResultSize) { + return findByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), query, expectedResultSize); + } + + protected PageData findByQueryAndCheck(CustomerId customerId, EntityDataQuery query, long expectedResultSize) { + PageData result = entityService.findEntityDataByQuery(tenantId, customerId, query); + assertThat(result.getTotalElements()).isEqualTo(expectedResultSize); + return result; + } + + protected List findByQueryAndCheckTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List expectedTelemetry) { + List loadedEntities = findEntitiesTelemetry(query, entityKeyType, key, expectedTelemetry); + List entitiesTelemetry = loadedEntities.stream().map(entityData -> entityData.getLatest().get(entityKeyType).get(key).getValue()).toList(); + assertThat(entitiesTelemetry).containsExactlyInAnyOrderElementsOf(expectedTelemetry); + return loadedEntities; + } + + protected List findEntitiesTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List expectedTelemetries) { + PageData data = findByQueryAndCheck(query, expectedTelemetries.size()); + List loadedEntities = new ArrayList<>(data.getData()); + while (data.hasNext()) { + query = query.next(); + data = findByQuery(query); + loadedEntities.addAll(data.getData()); + } + return loadedEntities; + } + + protected long countByQuery(CustomerId customerId, EntityCountQuery query) { + return entityService.countEntitiesByQuery(tenantId, customerId, query); + } + + protected long countByQueryAndCheck(EntityCountQuery countQuery, int expectedResult) { + return countByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), countQuery, expectedResult); + } + + protected long countByQueryAndCheck(CustomerId customerId, EntityCountQuery query, int expectedResult) { + long result = countByQuery(customerId, query); + assertThat(result).isEqualTo(expectedResult); + return result; + } + } diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java index b7f1f62582..ac85b48bc1 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java @@ -38,10 +38,17 @@ import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; +import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; +import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.entitiy.TbLogEntityActionService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; @@ -81,6 +88,20 @@ public class DefaultTbAlarmServiceTest { protected TbClusterService tbClusterService; @MockBean private EntitiesVersionControlService vcService; + @MockBean + private AccessControlService accessControlService; + @MockBean + private TenantService tenantService; + @MockBean + private AssetService assetService; + @MockBean + private DeviceService deviceService; + @MockBean + private AssetProfileService assetProfileService; + @MockBean + private DeviceProfileService deviceProfileService; + @MockBean + private EntityService entityService; @SpyBean DefaultTbAlarmService service; diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java index 0196857b1e..3c00c2957a 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java @@ -35,10 +35,17 @@ import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.dao.alarm.AlarmCommentService; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; +import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.entitiy.TbLogEntityActionService; import org.thingsboard.server.service.entitiy.alarm.DefaultTbAlarmCommentService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; import java.util.UUID; @@ -72,6 +79,20 @@ public class DefaultTbAlarmCommentServiceTest { protected CustomerService customerService; @MockBean protected TbClusterService tbClusterService; + @MockBean + private AccessControlService accessControlService; + @MockBean + private TenantService tenantService; + @MockBean + private AssetService assetService; + @MockBean + private DeviceService deviceService; + @MockBean + private AssetProfileService assetProfileService; + @MockBean + private DeviceProfileService deviceProfileService; + @MockBean + private EntityService entityService; @SpyBean DefaultTbAlarmCommentService service; diff --git a/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java b/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java index 9b6c5e6779..40369e231a 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java @@ -44,6 +44,7 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueCallback; @@ -102,6 +103,8 @@ public class DefaultTbClusterServiceTest { protected TbRuleEngineProducerService ruleEngineProducerService; @MockBean protected TbTransactionalCache edgeCache; + @MockBean + protected CalculatedFieldService calculatedFieldService; @SpyBean protected TopicService topicService; diff --git a/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerServiceTest.java b/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerServiceTest.java index fe222237c6..26832f6176 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerServiceTest.java @@ -532,6 +532,98 @@ public class DefaultTbCoreConsumerServiceTest { then(statsMock).should(never()).log(inactivityMsg); } + @Test + public void givenProcessingSuccess_whenForwardingInactivityTimeoutUpdateMsgToStateService_thenOnSuccessCallbackIsCalled() { + // GIVEN + var inactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) + .setInactivityTimeout(time) + .build(); + + doCallRealMethod().when(defaultTbCoreConsumerServiceMock).forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock); + + // WHEN + defaultTbCoreConsumerServiceMock.forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock); + + // THEN + then(stateServiceMock).should().onDeviceInactivityTimeoutUpdate(tenantId, deviceId, time); + then(tbCallbackMock).should().onSuccess(); + then(tbCallbackMock).should(never()).onFailure(any()); + } + + @Test + public void givenProcessingFailure_whenForwardingInactivityTimeoutUpdateMsgToStateService_thenOnFailureCallbackIsCalled() { + // GIVEN + var inactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) + .setInactivityTimeout(time) + .build(); + + doCallRealMethod().when(defaultTbCoreConsumerServiceMock).forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock); + + var runtimeException = new RuntimeException("Something bad happened!"); + doThrow(runtimeException).when(stateServiceMock).onDeviceInactivityTimeoutUpdate(tenantId, deviceId, time); + + // WHEN + defaultTbCoreConsumerServiceMock.forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock); + + // THEN + then(tbCallbackMock).should(never()).onSuccess(); + then(tbCallbackMock).should().onFailure(runtimeException); + } + + @Test + public void givenStatsEnabled_whenForwardingInactivityTimeoutUpdateMsgToStateService_thenStatsAreRecorded() { + // GIVEN + ReflectionTestUtils.setField(defaultTbCoreConsumerServiceMock, "stats", statsMock); + ReflectionTestUtils.setField(defaultTbCoreConsumerServiceMock, "statsEnabled", true); + + var inactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) + .setInactivityTimeout(time) + .build(); + + doCallRealMethod().when(defaultTbCoreConsumerServiceMock).forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock); + + // WHEN + defaultTbCoreConsumerServiceMock.forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock); + + // THEN + then(statsMock).should().log(inactivityTimeoutUpdateMsg); + } + + @Test + public void givenStatsDisabled_whenForwardingInactivityTimeoutUpdateMsgToStateService_thenStatsAreNotRecorded() { + // GIVEN + ReflectionTestUtils.setField(defaultTbCoreConsumerServiceMock, "stats", statsMock); + ReflectionTestUtils.setField(defaultTbCoreConsumerServiceMock, "statsEnabled", false); + + var inactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) + .setInactivityTimeout(time) + .build(); + + doCallRealMethod().when(defaultTbCoreConsumerServiceMock).forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock); + + // WHEN + defaultTbCoreConsumerServiceMock.forwardToStateService(inactivityTimeoutUpdateMsg, tbCallbackMock); + + // THEN + then(statsMock).should(never()).log(inactivityTimeoutUpdateMsg); + } + @Test public void givenRestApiCallResponseMsgProto_whenForwardToRuleEngineCallService_thenCallOnQueueMsg() { // GIVEN @@ -545,4 +637,5 @@ public class DefaultTbCoreConsumerServiceTest { // THEN then(ruleEngineCallServiceMock).should().onQueueMsg(restApiCallResponseMsgProto, tbCallbackMock); } + } diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java index 8141b924f6..314c76b058 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java @@ -626,8 +626,7 @@ public class TbRuleEngineQueueConsumerManagerTest { .until(() -> consumer.subscribed && consumer.getPartitions().equals(expectedPartitions) && consumer.pollingStarted); verify(consumer, times(1)).subscribe(any()); verify(consumer).subscribe(eq(expectedPartitions)); - verify(consumer).doSubscribe(argThat(topics -> topics.containsAll(expectedPartitions.stream() - .map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList())))); + verify(consumer).doSubscribe(argThat(topics -> topics.containsAll(expectedPartitions))); verify(consumer, atLeastOnce()).poll(eq((long) queue.getPollInterval())); verify(consumer, atLeastOnce()).doPoll(eq((long) queue.getPollInterval())); verify(consumer, never()).unsubscribe(); @@ -743,9 +742,11 @@ public class TbRuleEngineQueueConsumerManagerTest { } @Override - protected void doSubscribe(List topicNames) { - log.debug("doSubscribe({})", topicNames); - this.topics = topicNames; + protected void doSubscribe(Set partitions) { + this.topics = partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .collect(Collectors.toList()); + log.debug("doSubscribe({})", topics); subscribed = true; } diff --git a/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java b/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java index 2ec917c81a..fe416eacd5 100644 --- a/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java @@ -16,15 +16,22 @@ package org.thingsboard.server.service.resource.sql; import com.datastax.oss.driver.api.core.uuid.Uuids; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.jupiter.api.Assertions; import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.DashboardInfo; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityInfo; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TbResourceDeleteResult; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.TbResourceInfoFilter; import org.thingsboard.server.common.data.Tenant; @@ -36,10 +43,14 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.widget.WidgetTypeDetails; +import org.thingsboard.server.common.data.widget.WidgetTypeInfo; import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.service.resource.TbResourceService; import java.util.ArrayList; @@ -108,6 +119,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { ""; private static final String DEFAULT_FILE_NAME = "test.jks"; + private static final String JS_FILE_NAME = "test.js"; private static final String TEST_BASE64_DATA = "77u/PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPCEtLQpGSUxFIElORk9STUFUSU9OCgpPTUEgUGVybWFuZW50IERvY3VtZW50CiAgIEZpbGU6IE9NQS1TVVAtTHdNMk1fQmluYXJ5QXBwRGF0YUNvbnRhaW5lci1WMV8wXzEtMjAxOTAyMjEtQQogICBUeXBlOiB4bWwKClB1YmxpYyBSZWFjaGFibGUgSW5mb3JtYXRpb24KICAgUGF0aDogaHR0cDovL3d3dy5vcGVubW9iaWxlYWxsaWFuY2Uub3JnL3RlY2gvcHJvZmlsZXMKICAgTmFtZTogTHdNMk1fQmluYXJ5QXBwRGF0YUNvbnRhaW5lci12MV8wXzEueG1sCgpOT1JNQVRJVkUgSU5GT1JNQVRJT04KCiAgSW5mb3JtYXRpb24gYWJvdXQgdGhpcyBmaWxlIGNhbiBiZSBmb3VuZCBpbiB0aGUgbGF0ZXN0IHJldmlzaW9uIG9mCgogIE9NQS1UUy1MV00yTV9CaW5hcnlBcHBEYXRhQ29udGFpbmVyLVYxXzBfMQoKICBUaGlzIGlzIGF2YWlsYWJsZSBhdCBodHRwOi8vd3d3Lm9wZW5tb2JpbGVhbGxpYW5jZS5vcmcvCgogIFNlbmQgY29tbWVudHMgdG8gaHR0cHM6Ly9naXRodWIuY29tL09wZW5Nb2JpbGVBbGxpYW5jZS9PTUFfTHdNMk1fZm9yX0RldmVsb3BlcnMvaXNzdWVzCgpDSEFOR0UgSElTVE9SWQoKMTUwNjIwMTggU3RhdHVzIGNoYW5nZWQgdG8gQXBwcm92ZWQgYnkgRE0sIERvYyBSZWYgIyBPTUEtRE0mU0UtMjAxOC0wMDYxLUlOUF9MV00yTV9BUFBEQVRBX1YxXzBfRVJQX2Zvcl9maW5hbF9BcHByb3ZhbAoyMTAyMjAxOSBTdGF0dXMgY2hhbmdlZCB0byBBcHByb3ZlZCBieSBJUFNPLCBEb2MgUmVmICMgT01BLUlQU08tMjAxOS0wMDI1LUlOUF9Md00yTV9PYmplY3RfQXBwX0RhdGFfQ29udGFpbmVyXzFfMF8xX2Zvcl9GaW5hbF9BcHByb3ZhbAoKTEVHQUwgRElTQ0xBSU1FUgoKQ29weXJpZ2h0IDIwMTkgT3BlbiBNb2JpbGUgQWxsaWFuY2UuCgpSZWRpc3RyaWJ1dGlvbiBhbmQgdXNlIGluIHNvdXJjZSBhbmQgYmluYXJ5IGZvcm1zLCB3aXRoIG9yIHdpdGhvdXQKbW9kaWZpY2F0aW9uLCBhcmUgcGVybWl0dGVkIHByb3ZpZGVkIHRoYXQgdGhlIGZvbGxvd2luZyBjb25kaXRpb25zCmFyZSBtZXQ6CgoxLiBSZWRpc3RyaWJ1dGlvbnMgb2Ygc291cmNlIGNvZGUgbXVzdCByZXRhaW4gdGhlIGFib3ZlIGNvcHlyaWdodApub3RpY2UsIHRoaXMgbGlzdCBvZiBjb25kaXRpb25zIGFuZCB0aGUgZm9sbG93aW5nIGRpc2NsYWltZXIuCjIuIFJlZGlzdHJpYnV0aW9ucyBpbiBiaW5hcnkgZm9ybSBtdXN0IHJlcHJvZHVjZSB0aGUgYWJvdmUgY29weXJpZ2h0Cm5vdGljZSwgdGhpcyBsaXN0IG9mIGNvbmRpdGlvbnMgYW5kIHRoZSBmb2xsb3dpbmcgZGlzY2xhaW1lciBpbiB0aGUKZG9jdW1lbnRhdGlvbiBhbmQvb3Igb3RoZXIgbWF0ZXJpYWxzIHByb3ZpZGVkIHdpdGggdGhlIGRpc3RyaWJ1dGlvbi4KMy4gTmVpdGhlciB0aGUgbmFtZSBvZiB0aGUgY29weXJpZ2h0IGhvbGRlciBub3IgdGhlIG5hbWVzIG9mIGl0cwpjb250cmlidXRvcnMgbWF5IGJlIHVzZWQgdG8gZW5kb3JzZSBvciBwcm9tb3RlIHByb2R1Y3RzIGRlcml2ZWQKZnJvbSB0aGlzIHNvZnR3YXJlIHdpdGhvdXQgc3BlY2lmaWMgcHJpb3Igd3JpdHRlbiBwZXJtaXNzaW9uLgoKVEhJUyBTT0ZUV0FSRSBJUyBQUk9WSURFRCBCWSBUSEUgQ09QWVJJR0hUIEhPTERFUlMgQU5EIENPTlRSSUJVVE9SUwoiQVMgSVMiIEFORCBBTlkgRVhQUkVTUyBPUiBJTVBMSUVEIFdBUlJBTlRJRVMsIElOQ0xVRElORywgQlVUIE5PVApMSU1JVEVEIFRPLCBUSEUgSU1QTElFRCBXQVJSQU5USUVTIE9GIE1FUkNIQU5UQUJJTElUWSBBTkQgRklUTkVTUwpGT1IgQSBQQVJUSUNVTEFSIFBVUlBPU0UgQVJFIERJU0NMQUlNRUQuIElOIE5PIEVWRU5UIFNIQUxMIFRIRQpDT1BZUklHSFQgSE9MREVSIE9SIENPTlRSSUJVVE9SUyBCRSBMSUFCTEUgRk9SIEFOWSBESVJFQ1QsIElORElSRUNULApJTkNJREVOVEFMLCBTUEVDSUFMLCBFWEVNUExBUlksIE9SIENPTlNFUVVFTlRJQUwgREFNQUdFUyAoSU5DTFVESU5HLApCVVQgTk9UIExJTUlURUQgVE8sIFBST0NVUkVNRU5UIE9GIFNVQlNUSVRVVEUgR09PRFMgT1IgU0VSVklDRVM7CkxPU1MgT0YgVVNFLCBEQVRBLCBPUiBQUk9GSVRTOyBPUiBCVVNJTkVTUyBJTlRFUlJVUFRJT04pIEhPV0VWRVIKQ0FVU0VEIEFORCBPTiBBTlkgVEhFT1JZIE9GIExJQUJJTElUWSwgV0hFVEhFUiBJTiBDT05UUkFDVCwgU1RSSUNUCkxJQUJJTElUWSwgT1IgVE9SVCAoSU5DTFVESU5HIE5FR0xJR0VOQ0UgT1IgT1RIRVJXSVNFKSBBUklTSU5HIElOCkFOWSBXQVkgT1VUIE9GIFRIRSBVU0UgT0YgVEhJUyBTT0ZUV0FSRSwgRVZFTiBJRiBBRFZJU0VEIE9GIFRIRQpQT1NTSUJJTElUWSBPRiBTVUNIIERBTUFHRS4KClRoZSBhYm92ZSBsaWNlbnNlIGlzIHVzZWQgYXMgYSBsaWNlbnNlIHVuZGVyIGNvcHlyaWdodCBvbmx5LiBQbGVhc2UKcmVmZXJlbmNlIHRoZSBPTUEgSVBSIFBvbGljeSBmb3IgcGF0ZW50IGxpY2Vuc2luZyB0ZXJtczoKaHR0cHM6Ly93d3cub21hc3BlY3dvcmtzLm9yZy9hYm91dC9pbnRlbGxlY3R1YWwtcHJvcGVydHktcmlnaHRzLwoKLS0+CjxMV00yTSB4bWxuczp4c2k9Imh0dHA6Ly93d3cudzMub3JnLzIwMDEvWE1MU2NoZW1hLWluc3RhbmNlIiB4c2k6bm9OYW1lc3BhY2VTY2hlbWFMb2NhdGlvbj0iaHR0cDovL29wZW5tb2JpbGVhbGxpYW5jZS5vcmcvdGVjaC9wcm9maWxlcy9MV00yTS54c2QiPgoJPE9iamVjdCBPYmplY3RUeXBlPSJNT0RlZmluaXRpb24iPgoJCTxOYW1lPkJpbmFyeUFwcERhdGFDb250YWluZXI8L05hbWU+CgkJPERlc2NyaXB0aW9uMT48IVtDREFUQVtUaGlzIEx3TTJNIE9iamVjdHMgcHJvdmlkZXMgdGhlIGFwcGxpY2F0aW9uIHNlcnZpY2UgZGF0YSByZWxhdGVkIHRvIGEgTHdNMk0gU2VydmVyLCBlZy4gV2F0ZXIgbWV0ZXIgZGF0YS4gClRoZXJlIGFyZSBzZXZlcmFsIG1ldGhvZHMgdG8gY3JlYXRlIGluc3RhbmNlIHRvIGluZGljYXRlIHRoZSBtZXNzYWdlIGRpcmVjdGlvbiBiYXNlZCBvbiB0aGUgbmVnb3RpYXRpb24gYmV0d2VlbiBBcHBsaWNhdGlvbiBhbmQgTHdNMk0uIFRoZSBDbGllbnQgYW5kIFNlcnZlciBzaG91bGQgbmVnb3RpYXRlIHRoZSBpbnN0YW5jZShzKSB1c2VkIHRvIGV4Y2hhbmdlIHRoZSBkYXRhLiBGb3IgZXhhbXBsZToKIC0gVXNpbmcgYSBzaW5nbGUgaW5zdGFuY2UgZm9yIGJvdGggZGlyZWN0aW9ucyBjb21tdW5pY2F0aW9uLCBmcm9tIENsaWVudCB0byBTZXJ2ZXIgYW5kIGZyb20gU2VydmVyIHRvIENsaWVudC4KIC0gVXNpbmcgYW4gaW5zdGFuY2UgZm9yIGNvbW11bmljYXRpb24gZnJvbSBDbGllbnQgdG8gU2VydmVyIGFuZCBhbm90aGVyIG9uZSBmb3IgY29tbXVuaWNhdGlvbiBmcm9tIFNlcnZlciB0byBDbGllbnQKIC0gVXNpbmcgc2V2ZXJhbCBpbnN0YW5jZXMKXV0+PC9EZXNjcmlwdGlvbjE+CgkJPE9iamVjdElEPjE5PC9PYmplY3RJRD4KCQk8T2JqZWN0VVJOPnVybjpvbWE6bHdtMm06b21hOjE5PC9PYmplY3RVUk4+CgkJPExXTTJNVmVyc2lvbj4xLjA8L0xXTTJNVmVyc2lvbj4KCQk8T2JqZWN0VmVyc2lvbj4xLjA8L09iamVjdFZlcnNpb24+CgkJPE11bHRpcGxlSW5zdGFuY2VzPk11bHRpcGxlPC9NdWx0aXBsZUluc3RhbmNlcz4KCQk8TWFuZGF0b3J5Pk9wdGlvbmFsPC9NYW5kYXRvcnk+CgkJPFJlc291cmNlcz4KCQkJPEl0ZW0gSUQ9IjAiPjxOYW1lPkRhdGE8L05hbWU+CgkJCQk8T3BlcmF0aW9ucz5SVzwvT3BlcmF0aW9ucz4KCQkJCTxNdWx0aXBsZUluc3RhbmNlcz5NdWx0aXBsZTwvTXVsdGlwbGVJbnN0YW5jZXM+CgkJCQk8TWFuZGF0b3J5Pk1hbmRhdG9yeTwvTWFuZGF0b3J5PgoJCQkJPFR5cGU+T3BhcXVlPC9UeXBlPgoJCQkJPFJhbmdlRW51bWVyYXRpb24gLz4KCQkJCTxVbml0cyAvPgoJCQkJPERlc2NyaXB0aW9uPjwhW0NEQVRBW0luZGljYXRlcyB0aGUgYXBwbGljYXRpb24gZGF0YSBjb250ZW50Ll1dPjwvRGVzY3JpcHRpb24+CgkJCTwvSXRlbT4KCQkJPEl0ZW0gSUQ9IjEiPjxOYW1lPkRhdGEgUHJpb3JpdHk8L05hbWU+CgkJCQk8T3BlcmF0aW9ucz5SVzwvT3BlcmF0aW9ucz4KCQkJCTxNdWx0aXBsZUluc3RhbmNlcz5TaW5nbGU8L011bHRpcGxlSW5zdGFuY2VzPgoJCQkJPE1hbmRhdG9yeT5PcHRpb25hbDwvTWFuZGF0b3J5PgoJCQkJPFR5cGU+SW50ZWdlcjwvVHlwZT4KCQkJCTxSYW5nZUVudW1lcmF0aW9uPjEgYnl0ZXM8L1JhbmdlRW51bWVyYXRpb24+CgkJCQk8VW5pdHMgLz4KCQkJCTxEZXNjcmlwdGlvbj48IVtDREFUQVtJbmRpY2F0ZXMgdGhlIEFwcGxpY2F0aW9uIGRhdGEgcHJpb3JpdHk6CjA6SW1tZWRpYXRlCjE6QmVzdEVmZm9ydAoyOkxhdGVzdAozLTEwMDogUmVzZXJ2ZWQgZm9yIGZ1dHVyZSB1c2UuCjEwMS0yNTQ6IFByb3ByaWV0YXJ5IG1vZGUuXV0+PC9EZXNjcmlwdGlvbj4KCQkJPC9JdGVtPgoJCQk8SXRlbSBJRD0iMiI+PE5hbWU+RGF0YSBDcmVhdGlvbiBUaW1lPC9OYW1lPgoJCQkJPE9wZXJhdGlvbnM+Ulc8L09wZXJhdGlvbnM+CgkJCQk8TXVsdGlwbGVJbnN0YW5jZXM+U2luZ2xlPC9NdWx0aXBsZUluc3RhbmNlcz4KCQkJCTxNYW5kYXRvcnk+T3B0aW9uYWw8L01hbmRhdG9yeT4KCQkJCTxUeXBlPlRpbWU8L1R5cGU+CgkJCQk8UmFuZ2VFbnVtZXJhdGlvbiAvPgoJCQkJPFVuaXRzIC8+CgkJCQk8RGVzY3JpcHRpb24+PCFbQ0RBVEFbSW5kaWNhdGVzIHRoZSBEYXRhIGluc3RhbmNlIGNyZWF0aW9uIHRpbWVzdGFtcC5dXT48L0Rlc2NyaXB0aW9uPgoJCQk8L0l0ZW0+CgkJCTxJdGVtIElEPSIzIj48TmFtZT5EYXRhIERlc2NyaXB0aW9uPC9OYW1lPgoJCQkJPE9wZXJhdGlvbnM+Ulc8L09wZXJhdGlvbnM+CgkJCQk8TXVsdGlwbGVJbnN0YW5jZXM+U2luZ2xlPC9NdWx0aXBsZUluc3RhbmNlcz4KCQkJCTxNYW5kYXRvcnk+T3B0aW9uYWw8L01hbmRhdG9yeT4KCQkJCTxUeXBlPlN0cmluZzwvVHlwZT4KCQkJCTxSYW5nZUVudW1lcmF0aW9uPjMyIGJ5dGVzPC9SYW5nZUVudW1lcmF0aW9uPgoJCQkJPFVuaXRzIC8+CgkJCQk8RGVzY3JpcHRpb24+PCFbQ0RBVEFbSW5kaWNhdGVzIHRoZSBkYXRhIGRlc2NyaXB0aW9uLgplLmcuICJtZXRlciByZWFkaW5nIi5dXT48L0Rlc2NyaXB0aW9uPgoJCQk8L0l0ZW0+CgkJCTxJdGVtIElEPSI0Ij48TmFtZT5EYXRhIEZvcm1hdDwvTmFtZT4KCQkJCTxPcGVyYXRpb25zPlJXPC9PcGVyYXRpb25zPgoJCQkJPE11bHRpcGxlSW5zdGFuY2VzPlNpbmdsZTwvTXVsdGlwbGVJbnN0YW5jZXM+CgkJCQk8TWFuZGF0b3J5Pk9wdGlvbmFsPC9NYW5kYXRvcnk+CgkJCQk8VHlwZT5TdHJpbmc8L1R5cGU+CgkJCQk8UmFuZ2VFbnVtZXJhdGlvbj4zMiBieXRlczwvUmFuZ2VFbnVtZXJhdGlvbj4KCQkJCTxVbml0cyAvPgoJCQkJPERlc2NyaXB0aW9uPjwhW0NEQVRBW0luZGljYXRlcyB0aGUgZm9ybWF0IG9mIHRoZSBBcHBsaWNhdGlvbiBEYXRhLgplLmcuIFlHLU1ldGVyLVdhdGVyLVJlYWRpbmcKVVRGOC1zdHJpbmcKXV0+PC9EZXNjcmlwdGlvbj4KCQkJPC9JdGVtPgoJCQk8SXRlbSBJRD0iNSI+PE5hbWU+QXBwIElEPC9OYW1lPgoJCQkJPE9wZXJhdGlvbnM+Ulc8L09wZXJhdGlvbnM+CgkJCQk8TXVsdGlwbGVJbnN0YW5jZXM+U2luZ2xlPC9NdWx0aXBsZUluc3RhbmNlcz4KCQkJCTxNYW5kYXRvcnk+T3B0aW9uYWw8L01hbmRhdG9yeT4KCQkJCTxUeXBlPkludGVnZXI8L1R5cGU+CgkJCQk8UmFuZ2VFbnVtZXJhdGlvbj4yIGJ5dGVzPC9SYW5nZUVudW1lcmF0aW9uPgoJCQkJPFVuaXRzIC8+CgkJCQk8RGVzY3JpcHRpb24+PCFbQ0RBVEFbSW5kaWNhdGVzIHRoZSBkZXN0aW5hdGlvbiBBcHBsaWNhdGlvbiBJRC5dXT48L0Rlc2NyaXB0aW9uPgoJCQk8L0l0ZW0+PC9SZXNvdXJjZXM+CgkJPERlc2NyaXB0aW9uMj48IVtDREFUQVtdXT48L0Rlc2NyaXB0aW9uMj4KCTwvT2JqZWN0Pgo8L0xXTTJNPgo="; private static final byte[] TEST_DATA = Base64.getDecoder().decode(TEST_BASE64_DATA); @@ -119,6 +131,10 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { private ResourceService resourceService; @Autowired private TbResourceService tbResourceService; + @Autowired + private WidgetTypeService widgetTypeService; + @Autowired + private DashboardService dashboardService; private Tenant savedTenant; private User tenantAdmin; @@ -141,6 +157,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { tenantAdmin.setLastName("Downs"); tenantAdmin = createUserAndLogin(tenantAdmin, "testPassword1"); + } @After @@ -244,7 +261,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { assertEquals(title, foundResource.getTitle()); assertArrayEquals(foundResource.getData(), TEST_DATA); - tbResourceService.delete(foundResource, null); + tbResourceService.delete(foundResource, true, null); } @Test @@ -267,7 +284,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { assertEquals("0_1.0", foundResource.getResourceKey()); assertArrayEquals(foundResource.getData(), LWM2M_TEST_MODEL.getBytes()); - tbResourceService.delete(foundResource, null); + tbResourceService.delete(savedResource, true, null); } @Test @@ -281,8 +298,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { assertEquals(TenantId.SYS_TENANT_ID, savedResource.getTenantId()); - TbResource foundResource = resourceService.findResourceById(tenantId, savedResource.getId()); - tbResourceService.delete(foundResource, null); + tbResourceService.delete(savedResource, true, null); } @Test @@ -361,7 +377,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { Assert.assertNotNull(foundResource); assertEquals(savedResource, new TbResourceInfo(foundResource)); assertArrayEquals(TEST_DATA, foundResource.getData()); - tbResourceService.delete(foundResource, null); + tbResourceService.delete(foundResource, true, null); } @Test @@ -378,22 +394,207 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { Assert.assertNotNull(foundResource); assertEquals(savedResource, new TbResourceInfo(foundResource)); assertArrayEquals(TEST_DATA, foundResource.getData()); - tbResourceService.delete(foundResource, null); + tbResourceService.delete(foundResource, true, null); } @Test public void testDeleteResource() throws Exception { TbResource resource = new TbResource(); - resource.setResourceType(ResourceType.JKS); + resource.setResourceType(ResourceType.JS_MODULE); resource.setTitle("My resource"); resource.setFileName(DEFAULT_FILE_NAME); resource.setData(TEST_DATA); TbResourceInfo savedResource = tbResourceService.save(resource); - TbResource foundResource = resourceService.findResourceById(tenantId, savedResource.getId()); + TbResource foundResource = resourceService.findResourceById(savedTenant.getId(), savedResource.getId()); + Assert.assertNotNull(foundResource); + tbResourceService.delete(savedResource, true, null); + foundResource = resourceService.findResourceById(savedTenant.getId(), savedResource.getId()); + Assert.assertNull(foundResource); + } + + @Test + public void testUnForceDeleteResourceAssignWithWidget() throws Exception { + TbResource resource = new TbResource(); + resource.setResourceType(ResourceType.JS_MODULE); + resource.setTitle("My resource"); + resource.setFileName(JS_FILE_NAME); + resource.setTenantId(savedTenant.getId()); + resource.setData(TEST_DATA); + TbResourceInfo savedResource = tbResourceService.save(resource); + TbResource foundResource = resourceService.findResourceById(savedTenant.getId(), savedResource.getId()); + Assert.assertNotNull(foundResource); + String link = DataConstants.TB_RESOURCE_PREFIX + resource.getLink(); + + WidgetTypeDetails widgetTypeDetails = new WidgetTypeDetails(); + widgetTypeDetails.setTenantId(savedTenant.getId()); + widgetTypeDetails.setDescriptor(JacksonUtil.newObjectNode() + .put("sizeX", 3) + .put("sizeY", 3) + .put("resource", link) + .put("templateCss", "") + .put("controllerScript", "self.onInit = function() {\n self.ctx.$scope.actionWidget.onInit();\n}\n\nself.typeParameters = function() {\n return {\n previewWidth: '300px',\n previewHeight: '320px',\n embedTitlePanel: true,\n targetDeviceOptional: true,\n displayRpcMessageToast: false\n };\n};\n\nself.onDestroy = function() {\n}") + .put("settingsSchema", "") + .put("dataKeySettingsSchema", "{}\n") + .put("settingsDirective", "tb-scada-symbol-widget-settings") + .put("hasBasicMode", true) + .put("basicModeDirective", "tb-scada-symbol-basic-config")); + widgetTypeDetails.setName("Widget Type"); + + WidgetTypeDetails savedWidgetType = widgetTypeService.saveWidgetType(widgetTypeDetails); + WidgetTypeDetails foundWidgetType = widgetTypeService.findWidgetTypeDetailsById(savedTenant.getId(), savedWidgetType.getId()); + String resourceLink = foundWidgetType.getDescriptor().get("resource").asText(); + Assertions.assertNotNull(resourceLink); + Assert.assertEquals(resourceLink, link); + + TbResourceDeleteResult result = tbResourceService.delete(savedResource, false, null); + Assert.assertNotNull(result); + Assert.assertFalse(result.isSuccess()); + Assert.assertFalse(result.getReferences().isEmpty()); + Assert.assertEquals(1, result.getReferences().size()); + + WidgetTypeInfo widgetTypeInfo = (WidgetTypeInfo) result.getReferences().get(EntityType.WIDGET_TYPE.name()).get(0); + WidgetTypeInfo foundWidgetTypeInfo = new WidgetTypeInfo(foundWidgetType); + Assert.assertNotNull(widgetTypeInfo); + Assert.assertNotNull(foundWidgetTypeInfo); + Assert.assertEquals(widgetTypeInfo, foundWidgetTypeInfo); + + TbResourceInfo foundResourceInfo = resourceService.findResourceInfoById(savedTenant.getId(), savedResource.getId()); + Assert.assertNotNull(foundResource); + Assert.assertEquals(savedResource, foundResourceInfo); + } + + @Test + public void testForceDeleteResourceAssignWithWidget() throws Exception { + TbResource resource = new TbResource(); + resource.setResourceType(ResourceType.JS_MODULE); + resource.setTitle("My resource"); + resource.setFileName(JS_FILE_NAME); + resource.setTenantId(savedTenant.getId()); + resource.setData(TEST_DATA); + TbResourceInfo savedResource = tbResourceService.save(resource); + TbResource foundResource = resourceService.findResourceById(savedTenant.getId(), savedResource.getId()); + Assert.assertNotNull(foundResource); + String link = DataConstants.TB_RESOURCE_PREFIX + resource.getLink(); + + WidgetTypeDetails widgetTypeDetails = new WidgetTypeDetails(); + widgetTypeDetails.setTenantId(savedTenant.getId()); + widgetTypeDetails.setDescriptor(JacksonUtil.newObjectNode() + .put("type", "rpc") + .put("sizeX", 3) + .put("sizeY", 3) + .put("resource", link) + .put("templateCss", "") + .put("controllerScript", "self.onInit = function() {\n self.ctx.$scope.actionWidget.onInit();\n}\n\nself.typeParameters = function() {\n return {\n previewWidth: '300px',\n previewHeight: '320px',\n embedTitlePanel: true,\n targetDeviceOptional: true,\n displayRpcMessageToast: false\n };\n};\n\nself.onDestroy = function() {\n}") + .put("settingsSchema", "") + .put("dataKeySettingsSchema", "{}\n") + .put("settingsDirective", "tb-scada-symbol-widget-settings") + .put("hasBasicMode", true) + .put("basicModeDirective", "tb-scada-symbol-basic-config")); + widgetTypeDetails.setName("Widget Type"); + + WidgetTypeDetails savedWidgetType = widgetTypeService.saveWidgetType(widgetTypeDetails); + WidgetTypeDetails foundWidgetType = widgetTypeService.findWidgetTypeDetailsById(savedTenant.getId(), savedWidgetType.getId()); + String resourceLink = foundWidgetType.getDescriptor().get("resource").asText(); + Assertions.assertNotNull(resourceLink); + Assert.assertEquals(resourceLink, link); + + TbResourceDeleteResult result = tbResourceService.delete(savedResource, true, null); + Assert.assertNotNull(result); + Assert.assertTrue(result.isSuccess()); + Assert.assertNull(result.getReferences()); + + foundResource = resourceService.findResourceById(savedTenant.getId(), savedResource.getId()); + Assert.assertNull(foundResource); + } + + @Test + public void testUnForceDeleteResourceAssignWithDashboard() throws Exception { + TbResource resource = new TbResource(); + resource.setResourceType(ResourceType.JS_MODULE); + resource.setTitle("My resource"); + resource.setFileName(JS_FILE_NAME); + resource.setTenantId(savedTenant.getId()); + resource.setData(TEST_DATA); + TbResourceInfo savedResource = tbResourceService.save(resource); + TbResource foundResource = resourceService.findResourceById(savedTenant.getId(), savedResource.getId()); + Assert.assertNotNull(foundResource); + String link = DataConstants.TB_RESOURCE_PREFIX + resource.getLink(); + + Dashboard dashboard = new Dashboard(); + dashboard.setTitle("My dashboard"); + dashboard.setTenantId(savedTenant.getId()); + dashboard.setConfiguration(JacksonUtil.newObjectNode() + .put("widgets", """ + {"xxx": + {"config":{"actions":{"elementClick":[ + {"customResources":[{"url":{"entityType":"TB_RESOURCE","id": + "tb-resource;/api/resource/js_module/tenant/gateway-management-extension.js"},"isModule":true}, + {"url":"tb-resource;/api/resource/js_module/tenant/gateway-management-extension.js","isModule":true}]}]}}}} + """) + .put("someResource", link)); + + Dashboard savedDashboard = dashboardService.saveDashboard(dashboard); + Dashboard foundDashboard = dashboardService.findDashboardById(savedTenant.getId(), savedDashboard.getId()); + String resourceLink = foundDashboard.getConfiguration().get("someResource").asText(); + Assertions.assertNotNull(resourceLink); + Assert.assertEquals(resourceLink, link); + + TbResourceDeleteResult result = tbResourceService.delete(savedResource, false, null); + Assert.assertNotNull(result); + Assert.assertFalse(result.isSuccess()); + Assert.assertNotNull(result.getReferences()); + Assert.assertEquals(1, result.getReferences().size()); + + DashboardInfo dashboardInfo = (DashboardInfo) result.getReferences().get(EntityType.DASHBOARD.name()).get(0); + DashboardInfo foundDashboardInfo = dashboardService.findDashboardInfoById(savedTenant.getId(), savedDashboard.getId()); + Assert.assertNotNull(dashboardInfo); + Assert.assertNotNull(foundDashboardInfo); + Assert.assertEquals(foundDashboardInfo, dashboardInfo); + + foundResource = resourceService.findResourceById(savedTenant.getId(), savedResource.getId()); + Assert.assertNotNull(foundResource); + Assert.assertEquals(savedDashboard, foundDashboard); + } + + @Test + public void testForceDeleteResourceAssignWithDashboard() throws Exception { + TbResource resource = new TbResource(); + resource.setResourceType(ResourceType.JS_MODULE); + resource.setTitle("My resource"); + resource.setFileName(JS_FILE_NAME); + resource.setTenantId(savedTenant.getId()); + resource.setData(TEST_DATA); + TbResourceInfo savedResource = tbResourceService.save(resource); + TbResource foundResource = resourceService.findResourceById(savedTenant.getId(), savedResource.getId()); Assert.assertNotNull(foundResource); - tbResourceService.delete(foundResource, null); - foundResource = resourceService.findResourceById(tenantId, savedResource.getId()); + String link = DataConstants.TB_RESOURCE_PREFIX + resource.getLink(); + + Dashboard dashboard = new Dashboard(); + dashboard.setTitle("My dashboard"); + dashboard.setTenantId(savedTenant.getId()); + dashboard.setConfiguration(JacksonUtil.newObjectNode() + .set("widgets", JacksonUtil.toJsonNode(""" + {"xxx": + {"config":{"actions":{"elementClick":[ + {"customResources":[{"url":{"entityType":"TB_RESOURCE","id": + "tb-resource;/api/resource/js_module/tenant/gateway-management-extension.js"},"isModule":true}, + {"url":"tb-resource;/api/resource/js_module/tenant/gateway-management-extension.js","isModule":true}]}]}}}} + """)) + .put("someResource", link)); + + Dashboard savedDashboard = dashboardService.saveDashboard(dashboard); + Dashboard foundDashboard = dashboardService.findDashboardById(savedTenant.getId(), savedDashboard.getId()); + String resourceLink = foundDashboard.getConfiguration().get("someResource").asText(); + Assertions.assertNotNull(resourceLink); + Assert.assertEquals(resourceLink, link); + + TbResourceDeleteResult result = tbResourceService.delete(savedResource, true, null); + Assert.assertNotNull(result); + Assert.assertTrue(result.isSuccess()); + Assert.assertNull(result.getReferences()); + + foundResource = resourceService.findResourceById(savedTenant.getId(), savedResource.getId()); Assert.assertNull(foundResource); } diff --git a/application/src/test/java/org/thingsboard/server/service/script/TbelInvokeDocsIoTest.java b/application/src/test/java/org/thingsboard/server/service/script/TbelInvokeDocsIoTest.java index 8c8b82928a..bf86f98e6e 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/TbelInvokeDocsIoTest.java +++ b/application/src/test/java/org/thingsboard/server/service/script/TbelInvokeDocsIoTest.java @@ -21,6 +21,7 @@ import org.thingsboard.script.api.tbel.TbDate; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; import java.util.Base64; import java.util.Collections; import java.util.Comparator; @@ -146,6 +147,31 @@ class TbelInvokeDocsIoTest extends AbstractTbelInvokeTest { assertEquals(expected, actual); } + @Test + public void mapsImplicitIterationWithoutEntrySet() throws ExecutionException, InterruptedException { + msgStr = msgMapStr; + decoderStr = """ + foreach(element : msg){ + if(element.getKey() == null){ + return raiseError("Bad getKey"); + } + if(element.key == null){ + return raiseError("Bad key"); + } + if(element.getValue() == null){ + return raiseError("Bad getValue"); + } + if(element.value == null){ + return raiseError("Bad value"); + } + } + return msg; + """; + LinkedHashMap expected = expectedMap; + Object actual = invokeScript(evalScript(decoderStr), msgStr); + assertEquals(expected, actual); + } + @Test public void mapsGetInfoSize_Test() throws ExecutionException, InterruptedException { msgStr = msgMapStr; @@ -660,14 +686,13 @@ class TbelInvokeDocsIoTest extends AbstractTbelInvokeTest { """; decoderStr = """ var list = msg.list; - var listAdd = ["thigsboard", 4, 67]; return { list: list.clone(), length: list.length(), memorySize: list.memorySize(), indOf1: list.indexOf("B", 1), indOf2: list.indexOf(2, 2), - sStr: list.validateClazzInArrayIsOnlyString() + sStr: list.validateClazzInArrayIsOnlyNumber() } """; ArrayList list = new ArrayList<>(List.of(67, 2, 2, 2)); @@ -677,7 +702,7 @@ class TbelInvokeDocsIoTest extends AbstractTbelInvokeTest { expected.put("memorySize", 32L); expected.put("indOf1", -1); expected.put("indOf2", 2); - expected.put("sStr", false); + expected.put("sStr", true); Object actual = invokeScript(evalScript(decoderStr), msgStr); assertEquals(expected, actual); } @@ -2148,6 +2173,177 @@ class TbelInvokeDocsIoTest extends AbstractTbelInvokeTest { assertEquals(expected, actual); } + @Test + public void toUnmodifiableExecutionArrayList_Test() throws ExecutionException, InterruptedException { + msgStr = "{}"; + decoderStr = String.format(""" + var original = []; + original.add(0x35); + var unmodifiable = original.toUnmodifiable(); + msg.result = unmodifiable; + return {msg: msg}; + """); + LinkedHashMap expected = new LinkedHashMap<>(); + List expectedList = Arrays.asList(0x35); + LinkedHashMap expectedResult = new LinkedHashMap<>(); + expectedResult.put("result", expectedList); + expected.put("msg", expectedResult); + Object actual = invokeScript(evalScript(decoderStr), msgStr); + assertEquals(expected, actual); + + decoderStr = String.format(""" + var original = []; + original.add(0x67); + var unmodifiable = original.toUnmodifiable(); + unmodifiable.add(0x35); + msg.result = unmodifiable; + return {msg: msg}; + """); + assertThatThrownBy(() -> { + invokeScript(evalScript(decoderStr), msgStr); + }).hasMessageContaining("Error: unmodifiable.add(0x35): List is unmodifiable"); + } + + + @Test + public void toUnmodifiableExecutionHashMap_Test() throws ExecutionException, InterruptedException { + msgStr = "{}"; + decoderStr = String.format(""" + var original = {}; + original.putIfAbsent("entry1", 73); + var unmodifiable = original.toUnmodifiable(); + msg.result = unmodifiable; + return {msg: msg}; + """); + LinkedHashMap expected = new LinkedHashMap<>(); + LinkedHashMap expectedMap = new LinkedHashMap<>(Map.of("entry1", 73)); + LinkedHashMap expectedResult = new LinkedHashMap<>(); + expectedResult.put("result", expectedMap); + expected.put("msg", expectedResult); + Object actual = invokeScript(evalScript(decoderStr), msgStr); + assertEquals(expected, actual); + + decoderStr = String.format(""" + var original = {}; + original.humidity = 73; + var unmodifiable = original.toUnmodifiable(); + unmodifiable.put("temperature1", 96); + msg.result = unmodifiable; + return {msg: msg}; + """); + assertThatThrownBy(() -> { + invokeScript(evalScript(decoderStr), msgStr); + }).hasMessageContaining("Error: unmodifiable.put(\"temperature1\", 96): Map is unmodifiable"); + } + + @Test + public void tbDateFunction_Test() throws ExecutionException, InterruptedException { + String stringDateUTC = "2024-01-01T10:00:00.00Z"; + TbDate d = new TbDate(stringDateUTC); + + msgStr = "{}"; + decoderStr = String.format(""" + var d = new Date("%s"); // TZ => "UTC" + var dIsoY1 = d.toISOString(); // return 2024-01-01T10:00:00Z + d.addYears(1); + var dIsoY2 = d.toISOString(); // return 2025-01-01T10:00:00Z + d.addYears(-2); + var dIsoY3 = d.toISOString(); // return 2023-01-01T10:00:00Z + d.addMonths(2); + var dIsoM1 = d.toISOString(); // return 2023-03-01T10:00:00Z + d.addMonths(10); + var dIsoM2 = d.toISOString(); // return 2024-01-01T10:00:00Z + d.addMonths(-13); + var dIsoM3 = d.toISOString(); // return 2022-12-01T10:00:00Z + d.addWeeks(4); + var dIsoW1 = d.toISOString(); // return 2022-12-29T10:00:00Z + d.addWeeks(-5); + var dIsoW2 = d.toISOString(); // return 2022-11-24T10:00:00Z + d.addDays(6); + var dIsoD1 = d.toISOString(); // return 2022-11-30T10:00:00Z + d.addDays(45); + var dIsoD2 = d.toISOString(); // return 2023-01-14T10:00:00Z + d.addDays(-50); + var dIsoD3 = d.toISOString(); // return 2022-11-25T10:00:00Z + d.addHours(23); + var dIsoH1 = d.toISOString(); // return 2022-11-26T09:00:00Z + d.addHours(-47); + var dIsoH2 = d.toISOString(); // return 2022-11-24T10:00:00Z + d.addMinutes(59); + var dIsoMin1 = d.toISOString(); // return 2022-11-24T10:59:00Z + d.addMinutes(-60); + var dIsoMin2 = d.toISOString(); // return 2022-11-24T09:59:00Z + d.addSeconds(59); + var dIsoS1 = d.toISOString(); // return 2022-11-24T09:59:59Z + d.addSeconds(-60); + var dIsoS2 = d.toISOString(); // return 2022-11-24T09:58:59Z + d.addNanos(999999); + var dIsoN1 = d.toISOString(); // return 2022-11-24T09:58:59.000999999Z + d.addNanos(-1000000); + var dIsoN2 = d.toISOString(); // return 2022-11-24T09:58:58.999999999Z + return { + "dIsoY1": dIsoY1, + "dIsoY2": dIsoY2, + "dIsoY3": dIsoY3, + "dIsoM1": dIsoM1, + "dIsoM2": dIsoM2, + "dIsoM3": dIsoM3, + "dIsoW1": dIsoW1, + "dIsoW2": dIsoW2, + "dIsoD1": dIsoD1, + "dIsoD2": dIsoD2, + "dIsoD3": dIsoD3, + "dIsoH1": dIsoH1, + "dIsoH2": dIsoH2, + "dIsoMin1": dIsoMin1, + "dIsoMin2": dIsoMin2, + "dIsoS1": dIsoS1, + "dIsoS2": dIsoS2, + "dIsoN1": dIsoN1, + "dIsoN2": dIsoN2 + } + """, stringDateUTC); + LinkedHashMap expected = new LinkedHashMap<>(); + expected.put("dIsoY1", d.toISOString()); + d.addYears(1); + expected.put("dIsoY2", d.toISOString()); + d.addYears(-2); + expected.put("dIsoY3", d.toISOString()); + d.addMonths(2); + expected.put("dIsoM1", d.toISOString()); + d.addMonths(10); + expected.put("dIsoM2", d.toISOString()); + d.addMonths(-13); + expected.put("dIsoM3", d.toISOString()); + d.addWeeks(4); + expected.put("dIsoW1", d.toISOString()); + d.addWeeks(-5); + expected.put("dIsoW2", d.toISOString()); + d.addDays(6); + expected.put("dIsoD1", d.toISOString()); + d.addDays(45); + expected.put("dIsoD2", d.toISOString()); + d.addDays(-50); + expected.put("dIsoD3", d.toISOString()); + d.addHours(23); + expected.put("dIsoH1", d.toISOString()); + d.addHours(-47); + expected.put("dIsoH2", d.toISOString()); + d.addMinutes(59); + expected.put("dIsoMin1", d.toISOString()); + d.addMinutes(-60); + expected.put("dIsoMin2", d.toISOString()); + d.addSeconds(59); + expected.put("dIsoS1", d.toISOString()); + d.addSeconds(-60); + expected.put("dIsoS2", d.toISOString()); + d.addNanos(999999); + expected.put("dIsoN1", d.toISOString()); + d.addNanos(-1000000); + expected.put("dIsoN2", d.toISOString()); + Object actual = invokeScript(evalScript(decoderStr), msgStr); + assertEquals(expected, actual); + } private List splice(List oldList, int start, int deleteCount, Object... values) { start = initStartIndex(oldList, start); diff --git a/application/src/test/java/org/thingsboard/server/service/state/DefaultRuleEngineDeviceStateManagerTest.java b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateManagerTest.java similarity index 71% rename from application/src/test/java/org/thingsboard/server/service/state/DefaultRuleEngineDeviceStateManagerTest.java rename to application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateManagerTest.java index 28ea2d270a..6cf0ab9053 100644 --- a/application/src/test/java/org/thingsboard/server/service/state/DefaultRuleEngineDeviceStateManagerTest.java +++ b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateManagerTest.java @@ -52,7 +52,7 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.never; @ExtendWith(MockitoExtension.class) -public class DefaultRuleEngineDeviceStateManagerTest { +public class DefaultDeviceStateManagerTest { @Mock private DeviceStateService deviceStateServiceMock; @@ -71,7 +71,7 @@ public class DefaultRuleEngineDeviceStateManagerTest { @Captor private ArgumentCaptor queueCallbackCaptor; - private DefaultRuleEngineDeviceStateManager deviceStateManager; + private DefaultDeviceStateManager deviceStateManager; private static final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("57ab2e6c-bc4c-11ee-a506-0242ac120002")); private static final DeviceId DEVICE_ID = DeviceId.fromString("74a9053e-bc4c-11ee-a506-0242ac120002"); @@ -82,7 +82,7 @@ public class DefaultRuleEngineDeviceStateManagerTest { @BeforeEach public void setup() { - deviceStateManager = new DefaultRuleEngineDeviceStateManager(serviceInfoProviderMock, partitionServiceMock, Optional.of(deviceStateServiceMock), clusterServiceMock); + deviceStateManager = new DefaultDeviceStateManager(serviceInfoProviderMock, partitionServiceMock, Optional.of(deviceStateServiceMock), clusterServiceMock); } @ParameterizedTest @@ -90,7 +90,7 @@ public class DefaultRuleEngineDeviceStateManagerTest { "when onDeviceX() is called, then should route event to local service and call onSuccess() callback.") @MethodSource public void givenRoutedToLocalAndProcessingSuccess_whenOnDeviceAction_thenShouldCallLocalServiceAndSuccessCallback( - BiConsumer onDeviceAction, Consumer actionVerification + BiConsumer onDeviceAction, Consumer actionVerification ) { // GIVEN given(serviceInfoProviderMock.isService(ServiceType.TB_CORE)).willReturn(true); @@ -109,20 +109,24 @@ public class DefaultRuleEngineDeviceStateManagerTest { private static Stream givenRoutedToLocalAndProcessingSuccess_whenOnDeviceAction_thenShouldCallLocalServiceAndSuccessCallback() { return Stream.of( Arguments.of( - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS) ), Arguments.of( - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS) ), Arguments.of( - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS) ), Arguments.of( - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS) + ), + Arguments.of( + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS) ) ); } @@ -132,7 +136,7 @@ public class DefaultRuleEngineDeviceStateManagerTest { "when onDeviceX() is called, then should route event to local service and call onFailure() callback.") @MethodSource public void givenRoutedToLocalAndProcessingFailure_whenOnDeviceAction_thenShouldCallLocalServiceAndFailureCallback( - Consumer exceptionThrowSetup, BiConsumer onDeviceAction, Consumer actionVerification + Consumer exceptionThrowSetup, BiConsumer onDeviceAction, Consumer actionVerification ) { // GIVEN given(serviceInfoProviderMock.isService(ServiceType.TB_CORE)).willReturn(true); @@ -155,23 +159,28 @@ public class DefaultRuleEngineDeviceStateManagerTest { return Stream.of( Arguments.of( (Consumer) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS), - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS) ), Arguments.of( (Consumer) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS), - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS) ), Arguments.of( (Consumer) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS), - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS) ), Arguments.of( (Consumer) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS), - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS) + ), + Arguments.of( + (Consumer) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS) ) ); } @@ -181,7 +190,7 @@ public class DefaultRuleEngineDeviceStateManagerTest { "when onDeviceX() is called, then should send correct queue message to external service with correct callback object.") @MethodSource public void givenRoutedToExternal_whenOnDeviceAction_thenShouldSendQueueMsgToExternalServiceWithCorrectCallback( - BiConsumer onDeviceAction, BiConsumer> actionVerification + BiConsumer onDeviceAction, BiConsumer> actionVerification ) { // WHEN ReflectionTestUtils.setField(deviceStateManager, "deviceStateService", Optional.empty()); @@ -203,7 +212,7 @@ public class DefaultRuleEngineDeviceStateManagerTest { private static Stream givenRoutedToExternal_whenOnDeviceAction_thenShouldSendQueueMsgToExternalServiceWithCorrectCallback() { return Stream.of( Arguments.of( - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (BiConsumer>) (clusterServiceMock, queueCallbackCaptor) -> { var deviceConnectMsg = TransportProtos.DeviceConnectProto.newBuilder() .setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits()) @@ -219,7 +228,7 @@ public class DefaultRuleEngineDeviceStateManagerTest { } ), Arguments.of( - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (BiConsumer>) (clusterServiceMock, queueCallbackCaptor) -> { var deviceActivityMsg = TransportProtos.DeviceActivityProto.newBuilder() .setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits()) @@ -235,7 +244,7 @@ public class DefaultRuleEngineDeviceStateManagerTest { } ), Arguments.of( - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (BiConsumer>) (clusterServiceMock, queueCallbackCaptor) -> { var deviceDisconnectMsg = TransportProtos.DeviceDisconnectProto.newBuilder() .setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits()) @@ -251,7 +260,7 @@ public class DefaultRuleEngineDeviceStateManagerTest { } ), Arguments.of( - (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), (BiConsumer>) (clusterServiceMock, queueCallbackCaptor) -> { var deviceInactivityMsg = TransportProtos.DeviceInactivityProto.newBuilder() .setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits()) @@ -265,6 +274,22 @@ public class DefaultRuleEngineDeviceStateManagerTest { .build(); then(clusterServiceMock).should().pushMsgToCore(eq(EXTERNAL_TPI), any(UUID.class), eq(toCoreMsg), queueCallbackCaptor.capture()); } + ), + Arguments.of( + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivityTimeoutUpdate(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer>) (clusterServiceMock, queueCallbackCaptor) -> { + var deviceInactivityTimeoutUpdateMsg = TransportProtos.DeviceInactivityTimeoutUpdateProto.newBuilder() + .setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits()) + .setTenantIdLSB(TENANT_ID.getId().getLeastSignificantBits()) + .setDeviceIdMSB(DEVICE_ID.getId().getMostSignificantBits()) + .setDeviceIdLSB(DEVICE_ID.getId().getLeastSignificantBits()) + .setInactivityTimeout(EVENT_TS) + .build(); + var toCoreMsg = TransportProtos.ToCoreMsg.newBuilder() + .setDeviceInactivityTimeoutUpdateMsg(deviceInactivityTimeoutUpdateMsg) + .build(); + then(clusterServiceMock).should().pushMsgToCore(eq(EXTERNAL_TPI), any(UUID.class), eq(toCoreMsg), queueCallbackCaptor.capture()); + } ) ); } diff --git a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java index 26c978bbd0..26e913eacc 100644 --- a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java @@ -38,9 +38,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.notification.rule.trigger.DeviceActivityTrigger; import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.query.EntityData; -import org.thingsboard.server.common.data.query.EntityKeyType; -import org.thingsboard.server.common.data.query.TsValue; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; @@ -72,7 +69,7 @@ import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyCollection; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.BDDMockito.given; @@ -88,7 +85,6 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.thingsboard.server.service.state.DefaultDeviceStateService.ACTIVITY_STATE; import static org.thingsboard.server.service.state.DefaultDeviceStateService.INACTIVITY_ALARM_TIME; -import static org.thingsboard.server.service.state.DefaultDeviceStateService.INACTIVITY_TIMEOUT; import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAST_ACTIVITY_TIME; import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAST_CONNECT_TIME; import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAST_DISCONNECT_TIME; @@ -211,7 +207,7 @@ public class DefaultDeviceStateServiceTest { // THEN then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(LAST_CONNECT_TIME) && request.getEntries().get(0).getValue().equals(lastConnectTime) @@ -298,7 +294,7 @@ public class DefaultDeviceStateServiceTest { // THEN then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(LAST_DISCONNECT_TIME) && request.getEntries().get(0).getValue().equals(lastDisconnectTime) @@ -421,13 +417,13 @@ public class DefaultDeviceStateServiceTest { // THEN then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) && request.getEntries().get(0).getValue().equals(lastInactivityTime) )); then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && request.getEntries().get(0).getValue().equals(false) @@ -465,12 +461,12 @@ public class DefaultDeviceStateServiceTest { // THEN then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) )); then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && request.getEntries().get(0).getValue().equals(false) @@ -508,42 +504,6 @@ public class DefaultDeviceStateServiceTest { verify(service).fetchDeviceStateDataUsingSeparateRequests(deviceId); } - @Test - public void givenPersistToTelemetryAndDefaultInactivityTimeoutFetched_whenTransformingToDeviceStateData_thenTryGetInactivityFromAttribute() { - var defaultInactivityTimeoutInSec = 60L; - var latest = - Map.of( - EntityKeyType.TIME_SERIES, Map.of(INACTIVITY_TIMEOUT, new TsValue(0, Long.toString(defaultInactivityTimeoutInSec * 1000))), - EntityKeyType.SERVER_ATTRIBUTE, Map.of(INACTIVITY_TIMEOUT, new TsValue(0, Long.toString(5000L))) - ); - - process(latest, defaultInactivityTimeoutInSec); - } - - @Test - public void givenPersistToTelemetryAndNoInactivityTimeoutFetchedFromTimeSeries_whenTransformingToDeviceStateData_thenTryGetInactivityFromAttribute() { - var defaultInactivityTimeoutInSec = 60L; - var latest = - Map.of( - EntityKeyType.SERVER_ATTRIBUTE, Map.of(INACTIVITY_TIMEOUT, new TsValue(0, Long.toString(5000L))) - ); - - process(latest, defaultInactivityTimeoutInSec); - } - - private void process(Map> latest, long defaultInactivityTimeoutInSec) { - service.setDefaultInactivityTimeoutInSec(defaultInactivityTimeoutInSec); - service.setDefaultInactivityTimeoutMs(defaultInactivityTimeoutInSec * 1000); - service.setPersistToTelemetry(true); - - var deviceUuid = UUID.randomUUID(); - var deviceId = new DeviceId(deviceUuid); - - DeviceStateData deviceStateData = service.toDeviceStateData(new EntityData(deviceId, latest, Map.of()), new DeviceIdInfo(TenantId.SYS_TENANT_ID.getId(), UUID.randomUUID(), deviceUuid)); - - assertThat(deviceStateData.getState().getInactivityTimeout()).isEqualTo(5000L); - } - private void initStateService(long timeout) throws InterruptedException { service.stop(); reset(service, telemetrySubscriptionService); @@ -556,7 +516,7 @@ public class DefaultDeviceStateServiceTest { .thenReturn(new PageData<>(List.of(deviceIdInfo), 0, 1, false)); PartitionChangeEvent event = new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of( new QueueKey(ServiceType.TB_CORE), Collections.singleton(tpi) - )); + ), Collections.emptyMap()); service.onApplicationEvent(event); Thread.sleep(100); } @@ -1002,7 +962,7 @@ public class DefaultDeviceStateServiceTest { assertThat(actualNotification.isActive()).isFalse(); then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) && request.getEntries().get(0).getValue().equals(expectedLastInactivityAlarmTime) @@ -1114,7 +1074,7 @@ public class DefaultDeviceStateServiceTest { final long defaultTimeout = 1000; initStateService(defaultTimeout); given(deviceService.findDeviceById(any(TenantId.class), any(DeviceId.class))).willReturn(new Device(deviceId)); - given(attributesService.find(any(TenantId.class), any(EntityId.class), any(AttributeScope.class), anyList())).willReturn(Futures.immediateFuture(Collections.emptyList())); + given(attributesService.find(any(TenantId.class), any(EntityId.class), any(AttributeScope.class), anyCollection())).willReturn(Futures.immediateFuture(Collections.emptyList())); TransportProtos.DeviceStateServiceMsgProto proto = TransportProtos.DeviceStateServiceMsgProto.newBuilder() .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) @@ -1170,7 +1130,7 @@ public class DefaultDeviceStateServiceTest { assertThat(attributeRequestCaptor.getAllValues()).hasSize(2) .anySatisfy(request -> { - assertThat(request.getTenantId()).isEqualTo(TenantId.SYS_TENANT_ID); + assertThat(request.getTenantId()).isEqualTo(tenantId); assertThat(request.getEntityId()).isEqualTo(deviceId); assertThat(request.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE); assertThat(request.getEntries()).singleElement().satisfies(attributeKvEntry -> { @@ -1179,7 +1139,7 @@ public class DefaultDeviceStateServiceTest { }); }) .anySatisfy(request -> { - assertThat(request.getTenantId()).isEqualTo(TenantId.SYS_TENANT_ID); + assertThat(request.getTenantId()).isEqualTo(tenantId); assertThat(request.getEntityId()).isEqualTo(deviceId); assertThat(request.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE); assertThat(request.getEntries()).singleElement().satisfies(attributeKvEntry -> { @@ -1196,7 +1156,7 @@ public class DefaultDeviceStateServiceTest { final long defaultTimeout = 1000; initStateService(defaultTimeout); given(deviceService.findDeviceById(any(TenantId.class), any(DeviceId.class))).willReturn(new Device(deviceId)); - given(attributesService.find(any(TenantId.class), any(EntityId.class), any(AttributeScope.class), anyList())).willReturn(Futures.immediateFuture(Collections.emptyList())); + given(attributesService.find(any(TenantId.class), any(EntityId.class), any(AttributeScope.class), anyCollection())).willReturn(Futures.immediateFuture(Collections.emptyList())); long currentTime = System.currentTimeMillis(); DeviceState deviceState = DeviceState.builder() diff --git a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java index 7b4d6923f0..da3b214afb 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java @@ -43,6 +43,15 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.DeviceData; import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; @@ -70,6 +79,7 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.common.data.sync.ie.DeviceExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; @@ -79,6 +89,7 @@ import org.thingsboard.server.common.data.util.ThrowingRunnable; import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceProfileService; @@ -145,6 +156,8 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { protected TenantService tenantService; @Autowired protected EntityViewService entityViewService; + @Autowired + protected CalculatedFieldService calculatedFieldService; protected TenantId tenantId1; protected User tenantAdmin1; @@ -191,6 +204,7 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { Asset asset = createAsset(tenantId1, null, assetProfile.getId(), "Asset 1"); DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1"); Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device 1"); + CalculatedField calculatedField = createCalculatedField(tenantId1, device.getId(), asset.getId()); Map entitiesExportData = Stream.of(customer.getId(), asset.getId(), device.getId(), ruleChain.getId(), dashboard.getId(), assetProfile.getId(), deviceProfile.getId()) @@ -198,6 +212,7 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { try { return exportEntity(tenantAdmin1, entityId, EntityExportSettings.builder() .exportCredentials(false) + .exportCalculatedFields(true) .build()); } catch (Exception e) { throw new RuntimeException(e); @@ -245,7 +260,6 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { verify(entityActionService, Mockito.never()).logEntityAction(any(), eq(importedAsset.getId()), eq(importedAsset), any(), eq(ActionType.UPDATED), isNull()); - EntityExportData updatedAssetEntity = getAndClone(entitiesExportData, EntityType.ASSET); updatedAssetEntity.getEntity().setLabel("t" + updatedAssetEntity.getEntity().getLabel()); Asset updatedAsset = importEntity(tenantAdmin2, updatedAssetEntity).getSavedEntity(); @@ -268,10 +282,27 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.DEVICE)); verify(tbClusterService, Mockito.never()).onDeviceUpdated(eq(importedDevice), eq(importedDevice)); + // calculated field of imported device: + List calculatedFields = calculatedFieldService.findCalculatedFieldsByEntityId(tenantId2, importedDevice.getId()); + assertThat(calculatedFields.size()).isOne(); + var importedCalculatedField = calculatedFields.get(0); + assertThat(importedCalculatedField.getName()).isEqualTo(calculatedField.getName()); + verify(tbClusterService).onCalculatedFieldUpdated(eq(importedCalculatedField), isNull(), any()); + EntityExportData updatedDeviceEntity = getAndClone(entitiesExportData, EntityType.DEVICE); updatedDeviceEntity.getEntity().setLabel("t" + updatedDeviceEntity.getEntity().getLabel()); Device updatedDevice = importEntity(tenantAdmin2, updatedDeviceEntity).getSavedEntity(); verify(tbClusterService).onDeviceUpdated(eq(updatedDevice), eq(importedDevice)); + + // update calculated field: + DeviceExportData deviceExportData = (DeviceExportData) getAndClone(entitiesExportData, EntityType.DEVICE); + deviceExportData.setCalculatedFields(deviceExportData.getCalculatedFields().stream().peek(field -> field.setName("t_" + field.getName())).toList()); + importEntity(tenantAdmin2, deviceExportData).getSavedEntity(); + + calculatedFields = calculatedFieldService.findCalculatedFieldsByEntityId(tenantId2, importedDevice.getId()); + assertThat(calculatedFields.size()).isOne(); + importedCalculatedField = calculatedFields.get(0); + assertThat(importedCalculatedField.getName()).startsWith("t_"); } @Test @@ -290,12 +321,15 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { Device device = createDevice(tenantId1, customer.getId(), deviceProfile.getId(), "Device 1"); EntityView entityView = createEntityView(tenantId1, customer.getId(), device.getId(), "Entity view 1"); + CalculatedField calculatedField = createCalculatedField(tenantId1, device.getId(), device.getId()); + Map ids = new HashMap<>(); for (EntityId entityId : List.of(customer.getId(), ruleChain.getId(), dashboard.getId(), assetProfile.getId(), asset.getId(), deviceProfile.getId(), device.getId(), entityView.getId(), ruleChain.getId(), dashboard.getId())) { EntityExportData exportData = exportEntity(getSecurityUser(tenantAdmin1), entityId); EntityImportResult importResult = importEntity(getSecurityUser(tenantAdmin2), exportData, EntityImportSettings.builder() .saveCredentials(false) + .saveCalculatedFields(true) .build()); ids.put(entityId, (EntityId) importResult.getSavedEntity().getId()); } @@ -325,10 +359,16 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { assertThat(exportedDeviceProfile.getDefaultRuleChainId()).isEqualTo(ruleChain.getId()); assertThat(exportedDeviceProfile.getDefaultDashboardId()).isEqualTo(dashboard.getId()); - Device exportedDevice = (Device) exportEntity(tenantAdmin2, (DeviceId) ids.get(device.getId())).getEntity(); + EntityExportData entityExportData = exportEntity(tenantAdmin2, (DeviceId) ids.get(device.getId())); + Device exportedDevice = entityExportData.getEntity(); assertThat(exportedDevice.getCustomerId()).isEqualTo(customer.getId()); assertThat(exportedDevice.getDeviceProfileId()).isEqualTo(deviceProfile.getId()); + List calculatedFields = ((DeviceExportData) entityExportData).getCalculatedFields(); + assertThat(calculatedFields.size()).isOne(); + CalculatedField field = calculatedFields.get(0); + assertThat(field.getName()).isEqualTo(calculatedField.getName()); + EntityView exportedEntityView = (EntityView) exportEntity(tenantAdmin2, (EntityViewId) ids.get(entityView.getId())).getEntity(); assertThat(exportedEntityView.getCustomerId()).isEqualTo(customer.getId()); assertThat(exportedEntityView.getEntityId()).isEqualTo(device.getId()); @@ -340,7 +380,6 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { deviceProfileService.saveDeviceProfile(importedDeviceProfile); } - protected Device createDevice(TenantId tenantId, CustomerId customerId, DeviceProfileId deviceProfileId, String name) { Device device = new Device(); device.setTenantId(tenantId); @@ -549,9 +588,43 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { return relation; } + private CalculatedField createCalculatedField(TenantId tenantId, EntityId entityId, EntityId referencedEntityId) { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setEntityId(entityId); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); + calculatedField.setVersion(1L); + return calculatedFieldService.save(calculatedField); + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + argument.setRefEntityId(referencedEntityId); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + + config.setArguments(Map.of("T", argument)); + + config.setExpression("T - (100 - H) / 5"); + + Output output = new Output(); + output.setName("output"); + output.setType(OutputType.TIME_SERIES); + + config.setOutput(output); + + return config; + } + protected , I extends EntityId> EntityExportData exportEntity(User user, I entityId) throws Exception { return exportEntity(user, entityId, EntityExportSettings.builder() .exportCredentials(true) + .exportCalculatedFields(true) .build()); } @@ -562,6 +635,7 @@ public class ExportImportServiceSqlTest extends AbstractControllerTest { protected , I extends EntityId> EntityImportResult importEntity(User user, EntityExportData exportData) throws Exception { return importEntity(user, exportData, EntityImportSettings.builder() .saveCredentials(true) + .saveCalculatedFields(true) .build()); } diff --git a/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java index 0088ae2b7b..06f61ca4e5 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/vc/VersionControlTest.java @@ -44,6 +44,15 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.DeviceData; @@ -562,6 +571,81 @@ public class VersionControlTest extends AbstractControllerTest { checkImportedEntity(tenantId1, defaultDeviceProfile, tenantId2, importedDeviceProfile); } + @Test + public void testVcWithCalculatedFields_betweenTenants() throws Exception { + Asset asset = createAsset(null, null, "Asset 1"); + Device device = createDevice(null, null, "Device 1", "test1"); + CalculatedField calculatedField = createCalculatedField("CalculatedField1", device.getId(), asset.getId()); + String versionId = createVersion("calculated fields of asset and device", EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE); + + loginTenant2(); + loadVersion(versionId, config -> { + config.setLoadCredentials(false); + }, EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE); + + Asset importedAsset = findAsset(asset.getName()); + Device importedDevice = findDevice(device.getName()); + checkImportedEntity(tenantId1, device, tenantId2, importedDevice); + checkImportedEntity(tenantId1, asset, tenantId2, importedAsset); + + List importedCalculatedFields = findCalculatedFieldsByEntityId(importedDevice.getId()); + assertThat(importedCalculatedFields).size().isOne(); + assertThat(importedCalculatedFields.get(0)).satisfies(importedField -> { + assertThat(importedField.getName()).isEqualTo(calculatedField.getName()); + assertThat(importedField.getType()).isEqualTo(calculatedField.getType()); + assertThat(importedField.getId()).isNotEqualTo(calculatedField.getId()); + }); + } + + @Test + public void testVcWithReferencedCalculatedFields_betweenTenants() throws Exception { + Asset asset = createAsset(null, null, "Asset 1"); + Device device = createDevice(null, null, "Device 1", "test1"); + CalculatedField deviceCalculatedField = createCalculatedField("CalculatedField1", device.getId(), asset.getId()); + CalculatedField assetCalculatedField = createCalculatedField("CalculatedField2", asset.getId(), device.getId()); + String versionId = createVersion("calculated fields of asset and device", EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE); + + loginTenant2(); + loadVersion(versionId, config -> { + config.setLoadCredentials(false); + }, EntityType.ASSET, EntityType.DEVICE, EntityType.DEVICE_PROFILE, EntityType.ASSET_PROFILE); + + Asset importedAsset = findAsset(asset.getName()); + Device importedDevice = findDevice(device.getName()); + checkImportedEntity(tenantId1, device, tenantId2, importedDevice); + checkImportedEntity(tenantId1, asset, tenantId2, importedAsset); + + List importedDeviceCalculatedFields = findCalculatedFieldsByEntityId(importedDevice.getId()); + assertThat(importedDeviceCalculatedFields).size().isOne(); + assertThat(importedDeviceCalculatedFields.get(0)).satisfies(importedField -> { + assertThat(importedField.getName()).isEqualTo(deviceCalculatedField.getName()); + assertThat(importedField.getType()).isEqualTo(deviceCalculatedField.getType()); + assertThat(importedField.getId()).isNotEqualTo(deviceCalculatedField.getId()); + }); + + List importedAssetCalculatedFields = findCalculatedFieldsByEntityId(importedAsset.getId()); + assertThat(importedAssetCalculatedFields).size().isOne(); + assertThat(importedAssetCalculatedFields.get(0)).satisfies(importedField -> { + assertThat(importedField.getName()).isEqualTo(assetCalculatedField.getName()); + assertThat(importedField.getType()).isEqualTo(assetCalculatedField.getType()); + assertThat(importedField.getId()).isNotEqualTo(assetCalculatedField.getId()); + }); + } + + @Test + public void testVcWithCalculatedFields_sameTenant() throws Exception { + Asset asset = createAsset(null, null, "Asset 1"); + CalculatedField calculatedField = createCalculatedField("CalculatedField", asset.getId(), asset.getId()); + String versionId = createVersion("asset and field", EntityType.ASSET); + + loadVersion(versionId, EntityType.ASSET); + CalculatedField importedCalculatedField = findCalculatedFieldByEntityId(asset.getId()); + assertThat(importedCalculatedField.getId()).isEqualTo(calculatedField.getId()); + assertThat(importedCalculatedField.getName()).isEqualTo(calculatedField.getName()); + assertThat(importedCalculatedField.getConfiguration()).isEqualTo(calculatedField.getConfiguration()); + assertThat(importedCalculatedField.getType()).isEqualTo(calculatedField.getType()); + } + private & HasTenantId> void checkImportedEntity(TenantId tenantId1, E initialEntity, TenantId tenantId2, E importedEntity) { assertThat(initialEntity.getTenantId()).isEqualTo(tenantId1); assertThat(importedEntity.getTenantId()).isEqualTo(tenantId2); @@ -626,10 +710,10 @@ public class VersionControlTest extends AbstractControllerTest { request.setEntityTypes(Arrays.stream(entityTypes).collect(Collectors.toMap(t -> t, entityType -> { EntityTypeVersionCreateConfig config = new EntityTypeVersionCreateConfig(); config.setAllEntities(true); - config.setSaveRelations(true); config.setSaveAttributes(true); config.setSaveCredentials(true); + config.setSaveCalculatedFields(true); return config; }))); @@ -695,6 +779,7 @@ public class VersionControlTest extends AbstractControllerTest { config.setLoadAttributes(true); config.setLoadRelations(true); config.setLoadCredentials(true); + config.setLoadCalculatedFields(true); config.setRemoveOtherEntities(false); config.setFindExistingEntityByName(true); configModifier.accept(config); @@ -941,6 +1026,38 @@ public class VersionControlTest extends AbstractControllerTest { return doPost("/api/v2/relation", relation, EntityRelation.class); } + private CalculatedField createCalculatedField(String name, EntityId entityId, EntityId referencedEntityId) { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(entityId); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName(name); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); + calculatedField.setVersion(1L); + return doPost("/api/calculatedField", calculatedField, CalculatedField.class); + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + argument.setRefEntityId(referencedEntityId); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + + config.setArguments(Map.of("T", argument)); + + config.setExpression("T - (100 - H) / 5"); + + Output output = new Output(); + output.setName("output"); + output.setType(OutputType.TIME_SERIES); + + config.setOutput(output); + + return config; + } + protected void checkImportedRuleChainData(RuleChain initialRuleChain, RuleChainMetaData initialMetaData, RuleChain importedRuleChain, RuleChainMetaData importedMetaData) { assertThat(importedRuleChain.getType()).isEqualTo(initialRuleChain.getType()); assertThat(importedRuleChain.getName()).isEqualTo(initialRuleChain.getName()); @@ -995,11 +1112,18 @@ public class VersionControlTest extends AbstractControllerTest { private RuleChain findRuleChain(String name) throws Exception { return doGetTypedWithPageLink("/api/ruleChains?", new TypeReference>() {}, new PageLink(100, 0, name)).getData().get(0); - } private RuleChainMetaData findRuleChainMetaData(RuleChainId ruleChainId) throws Exception { return doGet("/api/ruleChain/" + ruleChainId + "/metadata", RuleChainMetaData.class); } + private CalculatedField findCalculatedFieldByEntityId(EntityId entityId) throws Exception { + return doGetTypedWithPageLink("/api/" + entityId.getEntityType() + "/" + entityId.getId() + "/calculatedFields?", new TypeReference>() {}, new PageLink(100, 0)).getData().get(0); + } + + private List findCalculatedFieldsByEntityId(EntityId entityId) throws Exception { + return doGetTypedWithPageLink("/api/" + entityId.getEntityType() + "/" + entityId.getId() + "/calculatedFields?", new TypeReference>() {}, new PageLink(100, 0)).getData(); + } + } diff --git a/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java index dbd61b638e..64845a12cd 100644 --- a/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java @@ -18,37 +18,50 @@ package org.thingsboard.server.service.telemetry; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; -import org.checkerframework.checker.nullness.qual.NonNull; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.rule.engine.api.AttributesDeleteRequest; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; +import org.thingsboard.rule.engine.api.DeviceStateManager; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.ApiUsageStateValue; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.id.ApiUsageStateId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.objects.AttributesEntityView; import org.thingsboard.server.common.data.objects.TelemetryEntityView; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.msg.rule.engine.DeviceAttributesEventNotificationMsg; import org.thingsboard.server.common.stats.TbApiUsageReportClient; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; @@ -56,6 +69,7 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldQueueService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.subscription.SubscriptionManagerService; @@ -71,11 +85,17 @@ import java.util.concurrent.ExecutorService; import java.util.stream.LongStream; import java.util.stream.Stream; +import static com.google.common.util.concurrent.Futures.immediateFailedFuture; import static com.google.common.util.concurrent.Futures.immediateFuture; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.BDDMockito.given; import static org.mockito.BDDMockito.then; import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; @ExtendWith(MockitoExtension.class) class DefaultTelemetrySubscriptionServiceTest { @@ -86,7 +106,7 @@ class DefaultTelemetrySubscriptionServiceTest { final long sampleTtl = 10_000L; - final List sampleTelemetry = List.of( + final List sampleTimeseries = List.of( new BasicTsKvEntry(100L, new DoubleDataEntry("temperature", 65.2)), new BasicTsKvEntry(100L, new DoubleDataEntry("humidity", 33.1)) ); @@ -98,14 +118,6 @@ class DefaultTelemetrySubscriptionServiceTest { .myPartition(true) .build(); - final FutureCallback emptyCallback = new FutureCallback<>() { - @Override - public void onSuccess(Void result) {} - - @Override - public void onFailure(@NonNull Throwable t) {} - }; - ExecutorService wsCallBackExecutor; ExecutorService tsCallBackExecutor; @@ -125,12 +137,16 @@ class DefaultTelemetrySubscriptionServiceTest { TbApiUsageReportClient apiUsageClient; @Mock TbApiUsageStateService apiUsageStateService; + @Mock + CalculatedFieldQueueService calculatedFieldQueueService; + @Mock + DeviceStateManager deviceStateManager; DefaultTelemetrySubscriptionService telemetryService; @BeforeEach void setup() { - telemetryService = new DefaultTelemetrySubscriptionService(attrService, tsService, tbEntityViewService, apiUsageClient, apiUsageStateService); + telemetryService = new DefaultTelemetrySubscriptionService(attrService, tsService, tbEntityViewService, apiUsageClient, apiUsageStateService, calculatedFieldQueueService, deviceStateManager); ReflectionTestUtils.setField(telemetryService, "clusterService", clusterService); ReflectionTestUtils.setField(telemetryService, "partitionService", partitionService); ReflectionTestUtils.setField(telemetryService, "subscriptionManagerService", Optional.of(subscriptionManagerService)); @@ -147,15 +163,22 @@ class DefaultTelemetrySubscriptionServiceTest { lenient().when(partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId)).thenReturn(tpi); - lenient().when(tsService.save(tenantId, entityId, sampleTelemetry, sampleTtl)).thenReturn(immediateFuture(sampleTelemetry.size())); - lenient().when(tsService.saveWithoutLatest(tenantId, entityId, sampleTelemetry, sampleTtl)).thenReturn(immediateFuture(sampleTelemetry.size())); - lenient().when(tsService.saveLatest(tenantId, entityId, sampleTelemetry)).thenReturn(immediateFuture(listOfNNumbers(sampleTelemetry.size()))); + lenient().when(tsService.save(tenantId, entityId, sampleTimeseries, sampleTtl)).thenReturn(immediateFuture(TimeseriesSaveResult.of(sampleTimeseries.size(), listOfNNumbers(sampleTimeseries.size())))); + lenient().when(tsService.saveWithoutLatest(tenantId, entityId, sampleTimeseries, sampleTtl)).thenReturn(immediateFuture(TimeseriesSaveResult.of(sampleTimeseries.size(), null))); + lenient().when(tsService.saveLatest(tenantId, entityId, sampleTimeseries)).thenReturn(immediateFuture(TimeseriesSaveResult.of(sampleTimeseries.size(), listOfNNumbers(sampleTimeseries.size())))); // mock no entity views lenient().when(tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId)).thenReturn(immediateFuture(Collections.emptyList())); + // mock that calls to CF queue service are always successful + lenient().doAnswer(inv -> { + FutureCallback callback = inv.getArgument(2); + callback.onSuccess(null); + return null; + }).when(calculatedFieldQueueService).pushRequestToQueue(any(TimeseriesSaveRequest.class), any(), any()); + // send partition change event so currentPartitions set is populated - telemetryService.onTbApplicationEvent(new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of(new QueueKey(ServiceType.TB_CORE), Set.of(tpi)))); + telemetryService.onTbApplicationEvent(new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of(new QueueKey(ServiceType.TB_CORE), Set.of(tpi)), Collections.emptyMap())); } @AfterEach @@ -164,6 +187,28 @@ class DefaultTelemetrySubscriptionServiceTest { tsCallBackExecutor.shutdownNow(); } + /* --- Save time series API --- */ + + @Test + void shouldThrowErrorWhenTryingToSaveTimeseriesForApiUsageState() { + // GIVEN + var request = TimeseriesSaveRequest.builder() + .tenantId(tenantId) + .customerId(customerId) + .entityId(new ApiUsageStateId(UUID.randomUUID())) + .entries(sampleTimeseries) + .strategy(TimeseriesSaveRequest.Strategy.PROCESS_ALL) + .build(); + + // WHEN + assertThatThrownBy(() -> telemetryService.saveTimeseries(request)) + .isInstanceOf(RuntimeException.class) + .hasMessage("Can't update API Usage State!"); + + // THEN + then(tsService).shouldHaveNoInteractions(); + } + @Test void shouldReportStorageDataPointsApiUsageWhenTimeSeriesIsSaved() { // GIVEN @@ -171,17 +216,16 @@ class DefaultTelemetrySubscriptionServiceTest { .tenantId(tenantId) .customerId(customerId) .entityId(entityId) - .entries(sampleTelemetry) + .entries(sampleTimeseries) .ttl(sampleTtl) - .strategy(new TimeseriesSaveRequest.Strategy(true, false, false)) - .callback(emptyCallback) + .strategy(new TimeseriesSaveRequest.Strategy(true, false, false, false)) .build(); // WHEN telemetryService.saveTimeseries(request); // THEN - then(apiUsageClient).should().report(tenantId, customerId, ApiUsageRecordKey.STORAGE_DP_COUNT, sampleTelemetry.size()); + then(apiUsageClient).should().report(tenantId, customerId, ApiUsageRecordKey.STORAGE_DP_COUNT, sampleTimeseries.size()); } @Test @@ -191,10 +235,9 @@ class DefaultTelemetrySubscriptionServiceTest { .tenantId(tenantId) .customerId(customerId) .entityId(entityId) - .entries(sampleTelemetry) + .entries(sampleTimeseries) .ttl(sampleTtl) .strategy(TimeseriesSaveRequest.Strategy.LATEST_AND_WS) - .callback(emptyCallback) .build(); // WHEN @@ -214,9 +257,9 @@ class DefaultTelemetrySubscriptionServiceTest { .tenantId(tenantId) .customerId(customerId) .entityId(entityId) - .entries(sampleTelemetry) + .entries(sampleTimeseries) .ttl(sampleTtl) - .strategy(TimeseriesSaveRequest.Strategy.SAVE_ALL) + .strategy(TimeseriesSaveRequest.Strategy.PROCESS_ALL) .future(future) .build(); @@ -240,7 +283,7 @@ class DefaultTelemetrySubscriptionServiceTest { .tenantId(tenantId) .customerId(customerId) .entityId(entityId) - .entries(sampleTelemetry) + .entries(sampleTimeseries) .ttl(sampleTtl) .strategy(TimeseriesSaveRequest.Strategy.LATEST_AND_WS) .future(future) @@ -260,12 +303,12 @@ class DefaultTelemetrySubscriptionServiceTest { entityView.setTenantId(tenantId); entityView.setCustomerId(customerId); entityView.setEntityId(entityId); - entityView.setKeys(new TelemetryEntityView(sampleTelemetry.stream().map(KvEntry::getKey).toList(), new AttributesEntityView())); + entityView.setKeys(new TelemetryEntityView(sampleTimeseries.stream().map(KvEntry::getKey).toList(), new AttributesEntityView())); // mock that there is one entity view given(tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId)).willReturn(immediateFuture(List.of(entityView))); // mock that save latest call for entity view is successful - given(tsService.saveLatest(tenantId, entityView.getId(), sampleTelemetry)).willReturn(immediateFuture(listOfNNumbers(sampleTelemetry.size()))); + given(tsService.saveLatest(tenantId, entityView.getId(), sampleTimeseries)).willReturn(immediateFuture(TimeseriesSaveResult.of(sampleTimeseries.size(), listOfNNumbers(sampleTimeseries.size())))); // mock TPI for entity view given(partitionService.resolve(ServiceType.TB_CORE, tenantId, entityView.getId())).willReturn(tpi); @@ -273,10 +316,9 @@ class DefaultTelemetrySubscriptionServiceTest { .tenantId(tenantId) .customerId(customerId) .entityId(entityId) - .entries(sampleTelemetry) + .entries(sampleTimeseries) .ttl(sampleTtl) - .strategy(new TimeseriesSaveRequest.Strategy(false, true, false)) - .callback(emptyCallback) + .strategy(new TimeseriesSaveRequest.Strategy(false, true, false, false)) .build(); // WHEN @@ -284,12 +326,12 @@ class DefaultTelemetrySubscriptionServiceTest { // THEN // should save latest to both the main entity and it's entity view - then(tsService).should().saveLatest(tenantId, entityId, sampleTelemetry); - then(tsService).should().saveLatest(tenantId, entityView.getId(), sampleTelemetry); + then(tsService).should().saveLatest(tenantId, entityId, sampleTimeseries); + then(tsService).should().saveLatest(tenantId, entityView.getId(), sampleTimeseries); then(tsService).shouldHaveNoMoreInteractions(); // should send WS update only for entity view (WS update for the main entity is disabled in the save request) - then(subscriptionManagerService).should().onTimeSeriesUpdate(tenantId, entityView.getId(), sampleTelemetry, TbCallback.EMPTY); + then(subscriptionManagerService).should().onTimeSeriesUpdate(tenantId, entityView.getId(), sampleTimeseries, TbCallback.EMPTY); then(subscriptionManagerService).shouldHaveNoMoreInteractions(); } @@ -300,10 +342,9 @@ class DefaultTelemetrySubscriptionServiceTest { .tenantId(tenantId) .customerId(customerId) .entityId(entityId) - .entries(sampleTelemetry) + .entries(sampleTimeseries) .ttl(sampleTtl) - .strategy(new TimeseriesSaveRequest.Strategy(true, false, false)) - .callback(emptyCallback) + .strategy(new TimeseriesSaveRequest.Strategy(true, false, false, false)) .build(); // WHEN @@ -311,7 +352,7 @@ class DefaultTelemetrySubscriptionServiceTest { // THEN // should save only time series for the main entity - then(tsService).should().saveWithoutLatest(tenantId, entityId, sampleTelemetry, sampleTtl); + then(tsService).should().saveWithoutLatest(tenantId, entityId, sampleTimeseries, sampleTtl); then(tsService).shouldHaveNoMoreInteractions(); // should not send any WS updates @@ -319,17 +360,16 @@ class DefaultTelemetrySubscriptionServiceTest { } @ParameterizedTest - @MethodSource("booleanCombinations") - void shouldCallCorrectApiBasedOnBooleanFlagsInTheSaveRequest(boolean saveTimeseries, boolean saveLatest, boolean sendWsUpdate) { + @MethodSource("allCombinationsOfFourBooleans") + void shouldCallCorrectSaveTimeseriesApiBasedOnBooleanFlagsInTheSaveRequest(boolean saveTimeseries, boolean saveLatest, boolean sendWsUpdate, boolean processCalculatedFields) { // GIVEN var request = TimeseriesSaveRequest.builder() .tenantId(tenantId) .customerId(customerId) .entityId(entityId) - .entries(sampleTelemetry) + .entries(sampleTimeseries) .ttl(sampleTtl) - .strategy(new TimeseriesSaveRequest.Strategy(saveTimeseries, saveLatest, sendWsUpdate)) - .callback(emptyCallback) + .strategy(new TimeseriesSaveRequest.Strategy(saveTimeseries, saveLatest, sendWsUpdate, processCalculatedFields)) .build(); // WHEN @@ -337,22 +377,86 @@ class DefaultTelemetrySubscriptionServiceTest { // THEN if (saveTimeseries && saveLatest) { - then(tsService).should().save(tenantId, entityId, sampleTelemetry, sampleTtl); + then(tsService).should().save(tenantId, entityId, sampleTimeseries, sampleTtl); } else if (saveLatest) { - then(tsService).should().saveLatest(tenantId, entityId, sampleTelemetry); + then(tsService).should().saveLatest(tenantId, entityId, sampleTimeseries); } else if (saveTimeseries) { - then(tsService).should().saveWithoutLatest(tenantId, entityId, sampleTelemetry, sampleTtl); + then(tsService).should().saveWithoutLatest(tenantId, entityId, sampleTimeseries, sampleTtl); + } + + if (processCalculatedFields) { + then(calculatedFieldQueueService).should().pushRequestToQueue(eq(request), any(), eq(request.getCallback())); } + then(tsService).shouldHaveNoMoreInteractions(); if (sendWsUpdate) { - then(subscriptionManagerService).should().onTimeSeriesUpdate(tenantId, entityId, sampleTelemetry, TbCallback.EMPTY); + then(subscriptionManagerService).should().onTimeSeriesUpdate(tenantId, entityId, sampleTimeseries, TbCallback.EMPTY); + } else { + then(subscriptionManagerService).shouldHaveNoInteractions(); + } + } + + private static Stream allCombinationsOfFourBooleans() { + return Stream.of( + Arguments.of(true, true, true, true), + Arguments.of(true, true, true, false), + Arguments.of(true, true, false, true), + Arguments.of(true, true, false, false), + Arguments.of(true, false, true, true), + Arguments.of(true, false, true, false), + Arguments.of(true, false, false, true), + Arguments.of(true, false, false, false), + Arguments.of(false, true, true, true), + Arguments.of(false, true, true, false), + Arguments.of(false, true, false, true), + Arguments.of(false, true, false, false), + Arguments.of(false, false, true, true), + Arguments.of(false, false, true, false), + Arguments.of(false, false, false, true), + Arguments.of(false, false, false, false) + ); + } + + /* --- Save attributes API --- */ + + @ParameterizedTest + @MethodSource("allCombinationsOfThreeBooleans") + void shouldCallCorrectSaveAttributesApiBasedOnBooleanFlagsInTheSaveRequest(boolean saveAttributes, boolean sendWsUpdate, boolean processCalculatedFields) { + // GIVEN + var request = AttributesSaveRequest.builder() + .tenantId(tenantId) + .entityId(entityId) + .scope(AttributeScope.SERVER_SCOPE) + .entry(new DoubleDataEntry("temperature", 65.2)) + .notifyDevice(false) + .strategy(new AttributesSaveRequest.Strategy(saveAttributes, sendWsUpdate, processCalculatedFields)) + .build(); + + lenient().when(attrService.save(tenantId, entityId, request.getScope(), request.getEntries())).thenReturn(immediateFuture(listOfNNumbers(request.getEntries().size()))); + + // WHEN + telemetryService.saveAttributes(request); + + // THEN + if (saveAttributes) { + then(attrService).should().save(tenantId, entityId, request.getScope(), request.getEntries()); + } else { + then(attrService).shouldHaveNoInteractions(); + } + + if (processCalculatedFields) { + then(calculatedFieldQueueService).should().pushRequestToQueue(eq(request), any(), eq(request.getCallback())); + } + + if (sendWsUpdate) { + then(subscriptionManagerService).should().onAttributesUpdate(tenantId, entityId, request.getScope().name(), request.getEntries(), TbCallback.EMPTY); } else { then(subscriptionManagerService).shouldHaveNoInteractions(); } } - private static Stream booleanCombinations() { + static Stream allCombinationsOfThreeBooleans() { return Stream.of( Arguments.of(true, true, true), Arguments.of(true, true, false), @@ -365,7 +469,642 @@ class DefaultTelemetrySubscriptionServiceTest { ); } - // used to emulate sequence numbers returned by save latest API + @Test + void shouldThrowErrorWhenTryingToSaveAttributesForApiUsageState() { + // GIVEN + var request = AttributesSaveRequest.builder() + .tenantId(tenantId) + .entityId(new ApiUsageStateId(UUID.randomUUID())) + .scope(AttributeScope.SHARED_SCOPE) + .entry(new DoubleDataEntry("temperature", 65.2)) + .notifyDevice(true) + .strategy(new AttributesSaveRequest.Strategy(true, false, false)) + .build(); + + // WHEN + assertThatThrownBy(() -> telemetryService.saveAttributes(request)) + .isInstanceOf(RuntimeException.class) + .hasMessage("Can't update API Usage State!"); + + // THEN + then(attrService).shouldHaveNoInteractions(); + } + + @Test + void shouldSendAttributesUpdateNotificationWhenDeviceSharedAttributesAreSavedAndNotifyDeviceIsTrue() { + // GIVEN + var deviceId = DeviceId.fromString("cc51e450-53e1-11ee-883e-e56b48fd2088"); + List entries = List.of( + new BaseAttributeKvEntry(123L, new DoubleDataEntry("shared1", 65.2)), + new BaseAttributeKvEntry(456L, new StringDataEntry("shared2", "test")) + ); + + var request = AttributesSaveRequest.builder() + .tenantId(tenantId) + .entityId(deviceId) + .scope(AttributeScope.SHARED_SCOPE) + .entries(entries) + .notifyDevice(true) + .strategy(new AttributesSaveRequest.Strategy(true, false, false)) + .build(); + + given(attrService.save(tenantId, deviceId, request.getScope(), entries)).willReturn(immediateFuture(listOfNNumbers(entries.size()))); + + // WHEN + telemetryService.saveAttributes(request); + + // THEN + var expectedAttributesUpdateMsg = DeviceAttributesEventNotificationMsg.onUpdate(tenantId, deviceId, "SHARED_SCOPE", entries); + + then(clusterService).should().pushMsgToCore(eq(expectedAttributesUpdateMsg), isNull()); + } + + @ParameterizedTest + @EnumSource( + value = EntityType.class, + names = {"DEVICE", "API_USAGE_STATE"}, // API usage state excluded due to coverage in another test + mode = EnumSource.Mode.EXCLUDE + ) + void shouldNotSendAttributesUpdateNotificationWhenEntityIsNotDevice(EntityType entityType) { + // GIVEN + var nonDeviceId = EntityIdFactory.getByTypeAndUuid(entityType, "cc51e450-53e1-11ee-883e-e56b48fd2088"); + List entries = List.of( + new BaseAttributeKvEntry(123L, new DoubleDataEntry("shared1", 65.2)), + new BaseAttributeKvEntry(456L, new StringDataEntry("shared2", "test")) + ); + + var request = AttributesSaveRequest.builder() + .tenantId(tenantId) + .entityId(nonDeviceId) + .scope(AttributeScope.SHARED_SCOPE) + .entries(entries) + .notifyDevice(true) + .strategy(new AttributesSaveRequest.Strategy(true, false, false)) + .build(); + + given(attrService.save(tenantId, nonDeviceId, request.getScope(), entries)).willReturn(immediateFuture(listOfNNumbers(entries.size()))); + + // WHEN + telemetryService.saveAttributes(request); + + // THEN + then(clusterService).should(never()).pushMsgToCore(any(), any()); + } + + @ParameterizedTest + @EnumSource( + value = AttributeScope.class, + names = "SHARED_SCOPE", + mode = EnumSource.Mode.EXCLUDE + ) + void shouldNotSendAttributesUpdateNotificationWhenAttributesAreNotShared(AttributeScope notSharedScope) { + // GIVEN + var deviceId = DeviceId.fromString("cc51e450-53e1-11ee-883e-e56b48fd2088"); + List entries = List.of( + new BaseAttributeKvEntry(123L, new DoubleDataEntry("shared1", 65.2)), + new BaseAttributeKvEntry(456L, new StringDataEntry("shared2", "test")) + ); + + var request = AttributesSaveRequest.builder() + .tenantId(tenantId) + .entityId(deviceId) + .scope(notSharedScope) + .entries(entries) + .notifyDevice(true) + .strategy(new AttributesSaveRequest.Strategy(true, false, false)) + .build(); + + given(attrService.save(tenantId, deviceId, request.getScope(), entries)).willReturn(immediateFuture(listOfNNumbers(entries.size()))); + + // WHEN + telemetryService.saveAttributes(request); + + // THEN + then(clusterService).should(never()).pushMsgToCore(any(), any()); + } + + @Test + void shouldNotSendAttributesUpdateNotificationWhenNotifyDeviceIsFalse() { + // GIVEN + var deviceId = DeviceId.fromString("cc51e450-53e1-11ee-883e-e56b48fd2088"); + List entries = List.of( + new BaseAttributeKvEntry(123L, new DoubleDataEntry("shared1", 65.2)), + new BaseAttributeKvEntry(456L, new StringDataEntry("shared2", "test")) + ); + + var request = AttributesSaveRequest.builder() + .tenantId(tenantId) + .entityId(deviceId) + .scope(AttributeScope.SHARED_SCOPE) + .entries(entries) + .notifyDevice(false) + .strategy(new AttributesSaveRequest.Strategy(true, false, false)) + .build(); + + given(attrService.save(tenantId, deviceId, request.getScope(), entries)).willReturn(immediateFuture(listOfNNumbers(entries.size()))); + + // WHEN + telemetryService.saveAttributes(request); + + // THEN + then(clusterService).should(never()).pushMsgToCore(any(), any()); + } + + @Test + void shouldNotSendAttributesUpdateNotificationWhenAttributesSaveWasSkipped() { + // GIVEN + var deviceId = DeviceId.fromString("cc51e450-53e1-11ee-883e-e56b48fd2088"); + List entries = List.of( + new BaseAttributeKvEntry(123L, new DoubleDataEntry("shared1", 65.2)), + new BaseAttributeKvEntry(456L, new StringDataEntry("shared2", "test")) + ); + + var request = AttributesSaveRequest.builder() + .tenantId(tenantId) + .entityId(deviceId) + .scope(AttributeScope.SHARED_SCOPE) + .entries(entries) + .notifyDevice(true) + .strategy(new AttributesSaveRequest.Strategy(false, false, false)) + .build(); + + // WHEN + telemetryService.saveAttributes(request); + + // THEN + then(clusterService).should(never()).pushMsgToCore(any(), any()); + } + + @Test + void shouldNotSendAttributesUpdateNotificationWhenAttributesSaveFailed() { + // GIVEN + var deviceId = DeviceId.fromString("cc51e450-53e1-11ee-883e-e56b48fd2088"); + List entries = List.of( + new BaseAttributeKvEntry(123L, new DoubleDataEntry("shared1", 65.2)), + new BaseAttributeKvEntry(456L, new StringDataEntry("shared2", "test")) + ); + + var request = AttributesSaveRequest.builder() + .tenantId(tenantId) + .entityId(deviceId) + .scope(AttributeScope.SHARED_SCOPE) + .entries(entries) + .notifyDevice(true) + .strategy(new AttributesSaveRequest.Strategy(true, false, false)) + .build(); + + given(attrService.save(tenantId, deviceId, request.getScope(), entries)).willReturn(immediateFailedFuture(new RuntimeException("failed to save"))); + + // WHEN + telemetryService.saveAttributes(request); + + // THEN + then(clusterService).should(never()).pushMsgToCore(any(), any()); + } + + @Test + void shouldNotifyDeviceStateManagerWhenDeviceInactivityTimeoutWasUpdated() { + // GIVEN + var deviceId = DeviceId.fromString("cc51e450-53e1-11ee-883e-e56b48fd2088"); + var inactivityTimeout = new BaseAttributeKvEntry(123L, new LongDataEntry("inactivityTimeout", 5000L)); + + var request = AttributesSaveRequest.builder() + .tenantId(tenantId) + .entityId(deviceId) + .scope(AttributeScope.SERVER_SCOPE) + .entry(inactivityTimeout) + .strategy(new AttributesSaveRequest.Strategy(true, false, false)) + .build(); + + given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())).willReturn(immediateFuture(listOfNNumbers(request.getEntries().size()))); + + // WHEN + telemetryService.saveAttributes(request); + + // THEN + then(deviceStateManager).should().onDeviceInactivityTimeoutUpdate(tenantId, deviceId, 5000L, TbCallback.EMPTY); + } + + @Test + void shouldNotNotifyDeviceStateManagerWhenDeviceInactivityTimeoutSaveWasSkipped() { + // GIVEN + var deviceId = DeviceId.fromString("cc51e450-53e1-11ee-883e-e56b48fd2088"); + var inactivityTimeout = new BaseAttributeKvEntry(123L, new LongDataEntry("inactivityTimeout", 5000L)); + + var request = AttributesSaveRequest.builder() + .tenantId(tenantId) + .entityId(deviceId) + .scope(AttributeScope.SERVER_SCOPE) + .entry(inactivityTimeout) + .strategy(new AttributesSaveRequest.Strategy(false, true, true)) + .build(); + + // WHEN + telemetryService.saveAttributes(request); + + // THEN + then(deviceStateManager).shouldHaveNoInteractions(); + } + + @ParameterizedTest + @EnumSource( + value = EntityType.class, + names = {"DEVICE", "API_USAGE_STATE"}, // API usage state excluded due to coverage in another test + mode = EnumSource.Mode.EXCLUDE + ) + void shouldNotNotifyDeviceStateManagerWhenInactivityTimeoutWasUpdatedButEntityTypeIsNotDevice(EntityType entityType) { + // GIVEN + var nonDeviceId = EntityIdFactory.getByTypeAndUuid(entityType, "cc51e450-53e1-11ee-883e-e56b48fd2088"); + var inactivityTimeout = new BaseAttributeKvEntry(123L, new LongDataEntry("inactivityTimeout", 5000L)); + + var request = AttributesSaveRequest.builder() + .tenantId(tenantId) + .entityId(nonDeviceId) + .scope(AttributeScope.SERVER_SCOPE) + .entry(inactivityTimeout) + .strategy(new AttributesSaveRequest.Strategy(true, false, false)) + .build(); + + given(attrService.save(tenantId, nonDeviceId, request.getScope(), request.getEntries())).willReturn(immediateFuture(listOfNNumbers(request.getEntries().size()))); + + // WHEN + telemetryService.saveAttributes(request); + + // THEN + then(deviceStateManager).shouldHaveNoInteractions(); + } + + @ParameterizedTest + @EnumSource( + value = AttributeScope.class, + names = {"SERVER_SCOPE"}, + mode = EnumSource.Mode.EXCLUDE + ) + void shouldNotNotifyDeviceStateManagerWhenInactivityTimeoutWasUpdatedButAttributeScopeIsNotServer(AttributeScope nonServerScope) { + // GIVEN + var deviceId = DeviceId.fromString("cc51e450-53e1-11ee-883e-e56b48fd2088"); + var inactivityTimeout = new BaseAttributeKvEntry(123L, new LongDataEntry("inactivityTimeout", 5000L)); + + var request = AttributesSaveRequest.builder() + .tenantId(tenantId) + .entityId(deviceId) + .scope(nonServerScope) + .entry(inactivityTimeout) + .strategy(new AttributesSaveRequest.Strategy(true, false, false)) + .build(); + + given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())).willReturn(immediateFuture(listOfNNumbers(request.getEntries().size()))); + + // WHEN + telemetryService.saveAttributes(request); + + // THEN + then(deviceStateManager).shouldHaveNoInteractions(); + } + + @Test + void shouldNotNotifyDeviceStateManagerWhenUpdatedAttributesDoNotContainInactivityTimeout() { + // GIVEN + var deviceId = DeviceId.fromString("cc51e450-53e1-11ee-883e-e56b48fd2088"); + var inactivityTimeout = new BaseAttributeKvEntry(123L, new LongDataEntry("notInactivityTimeout", 5000L)); + + var request = AttributesSaveRequest.builder() + .tenantId(tenantId) + .entityId(deviceId) + .scope(AttributeScope.SERVER_SCOPE) + .entry(inactivityTimeout) + .strategy(new AttributesSaveRequest.Strategy(true, false, false)) + .build(); + + given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())).willReturn(immediateFuture(listOfNNumbers(request.getEntries().size()))); + + // WHEN + telemetryService.saveAttributes(request); + + // THEN + then(deviceStateManager).shouldHaveNoInteractions(); + } + + @Test + void shouldUseInactivityTimeoutEntryWithTheGreatestVersion() { + // GIVEN + var deviceId = DeviceId.fromString("cc51e450-53e1-11ee-883e-e56b48fd2088"); + List entries = List.of( + new BaseAttributeKvEntry(new LongDataEntry("inactivityTimeout", 0L), 0L, null), + new BaseAttributeKvEntry(new LongDataEntry("inactivityTimeout", 1000L), 3L, 1L), + new BaseAttributeKvEntry(new LongDataEntry("inactivityTimeout", 2000L), 2L, 2L), + new BaseAttributeKvEntry(new LongDataEntry("inactivityTimeout", 3000L), 1L, 3L) + ); + + var request = AttributesSaveRequest.builder() + .tenantId(tenantId) + .entityId(deviceId) + .scope(AttributeScope.SERVER_SCOPE) + .entries(entries) + .strategy(new AttributesSaveRequest.Strategy(true, false, false)) + .build(); + + given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())).willReturn(immediateFuture(listOfNNumbers(request.getEntries().size()))); + + // WHEN + telemetryService.saveAttributes(request); + + // THEN + then(deviceStateManager).should().onDeviceInactivityTimeoutUpdate(tenantId, deviceId, 3000L, TbCallback.EMPTY); + } + + @Test + void shouldUseInactivityTimeoutEntryWithTheGreatestLastUpdateTsWhenVersionsAreTheSame() { + // GIVEN + var deviceId = DeviceId.fromString("cc51e450-53e1-11ee-883e-e56b48fd2088"); + List entries = List.of( + new BaseAttributeKvEntry(new LongDataEntry("inactivityTimeout", 1000L), 1L, 1L), + new BaseAttributeKvEntry(new LongDataEntry("inactivityTimeout", 2000L), 2L, 1L), + new BaseAttributeKvEntry(new LongDataEntry("inactivityTimeout", 3000L), 3L, 1L) + ); + + var request = AttributesSaveRequest.builder() + .tenantId(tenantId) + .entityId(deviceId) + .scope(AttributeScope.SERVER_SCOPE) + .entries(entries) + .strategy(new AttributesSaveRequest.Strategy(true, false, false)) + .build(); + + given(attrService.save(tenantId, deviceId, request.getScope(), request.getEntries())).willReturn(immediateFuture(listOfNNumbers(request.getEntries().size()))); + + // WHEN + telemetryService.saveAttributes(request); + + // THEN + then(deviceStateManager).should().onDeviceInactivityTimeoutUpdate(tenantId, deviceId, 3000L, TbCallback.EMPTY); + } + + /* --- Delete attributes API --- */ + + @Test + void shouldThrowErrorWhenTryingToDeleteAttributesForApiUsageState() { + // GIVEN + var request = AttributesDeleteRequest.builder() + .tenantId(tenantId) + .entityId(new ApiUsageStateId(UUID.randomUUID())) + .scope(AttributeScope.SHARED_SCOPE) + .keys(List.of("attributeKeyToDelete1", "attributeKeyToDelete2")) + .notifyDevice(true) + .build(); + + // WHEN + assertThatThrownBy(() -> telemetryService.deleteAttributes(request)) + .isInstanceOf(RuntimeException.class) + .hasMessage("Can't update API Usage State!"); + + // THEN + then(attrService).shouldHaveNoInteractions(); + } + + @Test + void shouldSendAttributesDeletedNotificationWhenDeviceSharedAttributesAreDeletedAndNotifyDeviceIsTrue() { + // GIVEN + var deviceId = DeviceId.fromString("cc51e450-53e1-11ee-883e-e56b48fd2088"); + List keys = List.of("attributeKeyToDelete1", "attributeKeyToDelete2"); + + var request = AttributesDeleteRequest.builder() + .tenantId(tenantId) + .entityId(deviceId) + .scope(AttributeScope.SHARED_SCOPE) + .keys(keys) + .notifyDevice(true) + .build(); + + given(attrService.removeAll(tenantId, deviceId, request.getScope(), keys)).willReturn(immediateFuture(keys)); + + // WHEN + telemetryService.deleteAttributes(request); + + // THEN + var expectedAttributesDeletedMsg = DeviceAttributesEventNotificationMsg.onDelete(tenantId, deviceId, "SHARED_SCOPE", List.of("attributeKeyToDelete1", "attributeKeyToDelete2")); + + then(clusterService).should().pushMsgToCore(eq(expectedAttributesDeletedMsg), isNull()); + } + + @ParameterizedTest + @EnumSource( + value = EntityType.class, + names = {"DEVICE", "API_USAGE_STATE"}, // API usage state excluded due to coverage in another test + mode = EnumSource.Mode.EXCLUDE + ) + void shouldNotSendAttributesDeletedNotificationWhenEntityIsNotDevice(EntityType entityType) { + // GIVEN + var nonDeviceId = EntityIdFactory.getByTypeAndUuid(entityType, "cc51e450-53e1-11ee-883e-e56b48fd2088"); + List keys = List.of("attributeKeyToDelete1", "attributeKeyToDelete2"); + + var request = AttributesDeleteRequest.builder() + .tenantId(tenantId) + .entityId(nonDeviceId) + .scope(AttributeScope.SHARED_SCOPE) + .keys(keys) + .notifyDevice(true) + .build(); + + given(attrService.removeAll(tenantId, nonDeviceId, request.getScope(), keys)).willReturn(immediateFuture(keys)); + + // WHEN + telemetryService.deleteAttributes(request); + + // THEN + then(clusterService).should(never()).pushMsgToCore(any(), any()); + } + + @ParameterizedTest + @EnumSource( + value = AttributeScope.class, + names = "SHARED_SCOPE", + mode = EnumSource.Mode.EXCLUDE + ) + void shouldNotSendAttributesDeletedNotificationWhenAttributesAreNotShared(AttributeScope notSharedScope) { + // GIVEN + var deviceId = DeviceId.fromString("cc51e450-53e1-11ee-883e-e56b48fd2088"); + List keys = List.of("attributeKeyToDelete1", "attributeKeyToDelete2"); + + var request = AttributesDeleteRequest.builder() + .tenantId(tenantId) + .entityId(deviceId) + .scope(notSharedScope) + .keys(keys) + .notifyDevice(true) + .build(); + + given(attrService.removeAll(tenantId, deviceId, request.getScope(), keys)).willReturn(immediateFuture(keys)); + + // WHEN + telemetryService.deleteAttributes(request); + + // THEN + then(clusterService).should(never()).pushMsgToCore(any(), any()); + } + + @Test + void shouldNotSendAttributesDeletedNotificationWhenNotifyDeviceIsFalse() { + // GIVEN + var deviceId = DeviceId.fromString("cc51e450-53e1-11ee-883e-e56b48fd2088"); + List keys = List.of("attributeKeyToDelete1", "attributeKeyToDelete2"); + + var request = AttributesDeleteRequest.builder() + .tenantId(tenantId) + .entityId(deviceId) + .scope(AttributeScope.SHARED_SCOPE) + .keys(keys) + .notifyDevice(false) + .build(); + + given(attrService.removeAll(tenantId, deviceId, request.getScope(), keys)).willReturn(immediateFuture(keys)); + + // WHEN + telemetryService.deleteAttributes(request); + + // THEN + then(clusterService).should(never()).pushMsgToCore(any(), any()); + } + + @Test + void shouldNotSendAttributesDeletedNotificationWhenAttributesDeleteFailed() { + // GIVEN + var deviceId = DeviceId.fromString("cc51e450-53e1-11ee-883e-e56b48fd2088"); + List keys = List.of("attributeKeyToDelete1", "attributeKeyToDelete2"); + + var request = AttributesDeleteRequest.builder() + .tenantId(tenantId) + .entityId(deviceId) + .scope(AttributeScope.SHARED_SCOPE) + .keys(keys) + .notifyDevice(true) + .build(); + + given(attrService.removeAll(tenantId, deviceId, request.getScope(), keys)).willReturn(immediateFailedFuture(new RuntimeException("failed to delete"))); + + // WHEN + telemetryService.deleteAttributes(request); + + // THEN + then(clusterService).should(never()).pushMsgToCore(any(), any()); + } + + @Test + void shouldNotifyDeviceStateManagerWhenDeviceInactivityTimeoutWasDeleted() { + // GIVEN + var deviceId = DeviceId.fromString("cc51e450-53e1-11ee-883e-e56b48fd2088"); + + var request = AttributesDeleteRequest.builder() + .tenantId(tenantId) + .entityId(deviceId) + .scope(AttributeScope.SERVER_SCOPE) + .keys(List.of("inactivityTimeout", "someOtherDeletedAttribute")) + .build(); + + given(attrService.removeAll(tenantId, deviceId, request.getScope(), request.getKeys())).willReturn(immediateFuture(request.getKeys())); + + // WHEN + telemetryService.deleteAttributes(request); + + // THEN + then(deviceStateManager).should().onDeviceInactivityTimeoutUpdate(tenantId, deviceId, 0L, TbCallback.EMPTY); + } + + @ParameterizedTest + @EnumSource( + value = EntityType.class, + names = {"DEVICE", "API_USAGE_STATE"}, // API usage state excluded due to coverage in another test + mode = EnumSource.Mode.EXCLUDE + ) + void shouldNotNotifyDeviceStateManagerWhenInactivityTimeoutWasDeletedButEntityTypeIsNotDevice(EntityType entityType) { + // GIVEN + var nonDeviceId = EntityIdFactory.getByTypeAndUuid(entityType, "cc51e450-53e1-11ee-883e-e56b48fd2088"); + + var request = AttributesDeleteRequest.builder() + .tenantId(tenantId) + .entityId(nonDeviceId) + .scope(AttributeScope.SERVER_SCOPE) + .keys(List.of("inactivityTimeout", "someOtherDeletedAttribute")) + .build(); + + given(attrService.removeAll(tenantId, nonDeviceId, request.getScope(), request.getKeys())).willReturn(immediateFuture(request.getKeys())); + + // WHEN + telemetryService.deleteAttributes(request); + + // THEN + then(deviceStateManager).shouldHaveNoInteractions(); + } + + @ParameterizedTest + @EnumSource( + value = AttributeScope.class, + names = {"SERVER_SCOPE"}, + mode = EnumSource.Mode.EXCLUDE + ) + void shouldNotNotifyDeviceStateManagerWhenInactivityTimeoutWasDeletedButAttributeScopeIsNotServer(AttributeScope nonServerScope) { + // GIVEN + var deviceId = DeviceId.fromString("cc51e450-53e1-11ee-883e-e56b48fd2088"); + + var request = AttributesDeleteRequest.builder() + .tenantId(tenantId) + .entityId(deviceId) + .scope(nonServerScope) + .keys(List.of("inactivityTimeout", "someOtherDeletedAttribute")) + .build(); + + given(attrService.removeAll(tenantId, deviceId, request.getScope(), request.getKeys())).willReturn(immediateFuture(request.getKeys())); + + // WHEN + telemetryService.deleteAttributes(request); + + // THEN + then(deviceStateManager).shouldHaveNoInteractions(); + } + + @Test + void shouldNotNotifyDeviceStateManagerWhenInactivityTimeoutWasNotDeleted() { + // GIVEN + var deviceId = DeviceId.fromString("cc51e450-53e1-11ee-883e-e56b48fd2088"); + + var request = AttributesDeleteRequest.builder() + .tenantId(tenantId) + .entityId(deviceId) + .scope(AttributeScope.SERVER_SCOPE) + .keys(List.of("someOtherDeletedAttribute")) + .build(); + + given(attrService.removeAll(tenantId, deviceId, request.getScope(), request.getKeys())).willReturn(immediateFuture(request.getKeys())); + + // WHEN + telemetryService.deleteAttributes(request); + + // THEN + then(deviceStateManager).shouldHaveNoInteractions(); + } + + @Test + void shouldNotNotifyDeviceStateManagerWhenDeviceInactivityTimeoutDeleteFailed() { + // GIVEN + var deviceId = DeviceId.fromString("cc51e450-53e1-11ee-883e-e56b48fd2088"); + + var request = AttributesDeleteRequest.builder() + .tenantId(tenantId) + .entityId(deviceId) + .scope(AttributeScope.SERVER_SCOPE) + .keys(List.of("inactivityTimeout", "someOtherDeletedAttribute")) + .build(); + + given(attrService.removeAll(tenantId, deviceId, request.getScope(), request.getKeys())).willReturn(immediateFailedFuture(new RuntimeException("failed to delete"))); + + // WHEN + telemetryService.deleteAttributes(request); + + // THEN + then(deviceStateManager).shouldHaveNoInteractions(); + } + + // used to emulate versions returned by save APIs private static List listOfNNumbers(int N) { return LongStream.range(0, N).boxed().toList(); } diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index a5cc50f5d1..b8bdcf67e6 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -35,6 +35,8 @@ sql.events.batch_threads=2 actors.system.tenant_dispatcher_pool_size=4 actors.system.device_dispatcher_pool_size=8 actors.system.rule_dispatcher_pool_size=12 +actors.system.cfm_dispatcher_pool_size=2 +actors.system.cfe_dispatcher_pool_size=2 transport.sessions.report_timeout=10000 queue.transport_api.request_poll_interval=5 queue.transport_api.response_poll_interval=5 @@ -56,3 +58,6 @@ sql.ttl.edge_events.edge_event_ttl=2592000 server.log_controller_error_stack_trace=false transport.gateway.dashboard.sync.enabled=false + +queue.edqs.sync.enabled=false +queue.edqs.api.supported=false diff --git a/application/src/test/resources/update/330/device_profile_001_out.json b/application/src/test/resources/update/330/device_profile_001_out.json index 9a349c6638..29e2241ee9 100644 --- a/application/src/test/resources/update/330/device_profile_001_out.json +++ b/application/src/test/resources/update/330/device_profile_001_out.json @@ -64,7 +64,8 @@ "dynamicValue": { "sourceType": null, "sourceAttribute": null, - "inherit": false + "inherit": false, + "resolvedValue" : null } } } @@ -103,7 +104,8 @@ "dynamicValue": { "sourceType": null, "sourceAttribute": null, - "inherit": false + "inherit": false, + "resolvedValue" : null } } } diff --git a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java index 741908a14c..4b822e0030 100644 --- a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java +++ b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java @@ -153,7 +153,7 @@ public final class TbActorMailbox implements TbActorCtx { } if (msg != null) { try { - log.debug("[{}] Going to process message: {}", selfId, msg); + log.trace("[{}] Going to process message: {}", selfId, msg); actor.process(msg); } catch (TbRuleNodeUpdateException updateException) { stopReason = TbActorStopReason.INIT_FAILED; diff --git a/common/actor/src/main/java/org/thingsboard/server/actors/TbCalculatedFieldEntityActorId.java b/common/actor/src/main/java/org/thingsboard/server/actors/TbCalculatedFieldEntityActorId.java new file mode 100644 index 0000000000..3b69fe7eff --- /dev/null +++ b/common/actor/src/main/java/org/thingsboard/server/actors/TbCalculatedFieldEntityActorId.java @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors; + +import lombok.Getter; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.Objects; + +public class TbCalculatedFieldEntityActorId implements TbActorId { + + @Getter + private final EntityId entityId; + + public TbCalculatedFieldEntityActorId(EntityId entityId) { + this.entityId = entityId; + } + + @Override + public String toString() { + return entityId.getEntityType() + "|" + entityId.getId(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TbCalculatedFieldEntityActorId that = (TbCalculatedFieldEntityActorId) o; + return entityId.equals(that.entityId); + } + + @Override + public int hashCode() { + // Magic number to ensure that the hash does not match with the hash of other actor id - (TbEntityActorId) + return 42 + Objects.hash(entityId); + } + + @Override + public EntityType getEntityType() { + return entityId.getEntityType(); + } +} diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 1962e2f147..2a2d04e0fd 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -21,6 +21,8 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.EdgeId; @@ -36,7 +38,10 @@ import org.thingsboard.server.common.msg.edge.ToEdgeSyncRequest; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.RestApiCallResponseMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; @@ -58,6 +63,8 @@ public interface TbClusterService extends TbQueueClusterService { void broadcastToCore(ToCoreNotificationMsg msg); + void broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg build, TbQueueCallback callback); + void pushMsgToVersionControl(TenantId tenantId, ToVersionControlServiceMsg msg, TbQueueCallback callback); void pushNotificationToCore(String targetServiceId, FromDeviceRpcResponse response, TbQueueCallback callback); @@ -74,6 +81,12 @@ public interface TbClusterService extends TbQueueClusterService { void pushNotificationToTransport(String targetServiceId, ToTransportMsg response, TbQueueCallback callback); + void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, TransportProtos.ToCalculatedFieldMsg msg, TbQueueCallback callback); + + void pushMsgToCalculatedFields(TopicPartitionInfo tpi, UUID msgId, ToCalculatedFieldMsg msg, TbQueueCallback callback); + + void pushNotificationToCalculatedFields(TenantId tenantId, EntityId entityId, TransportProtos.ToCalculatedFieldNotificationMsg msg, TbQueueCallback callback); + void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); void onDeviceProfileChange(DeviceProfile deviceProfile, DeviceProfile oldDeviceProfile, TbQueueCallback callback); @@ -96,6 +109,10 @@ public interface TbClusterService extends TbQueueClusterService { void onDeviceAssignedToTenant(TenantId oldTenantId, Device device); + void onAssetUpdated(Asset asset, Asset old); + + void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback); + void onResourceChange(TbResourceInfo resource, TbQueueCallback callback); void onResourceDeleted(TbResourceInfo resource, TbQueueCallback callback); @@ -114,4 +131,8 @@ public interface TbClusterService extends TbQueueClusterService { void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId sourceEdgeId); + void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, TbQueueCallback callback); + + void onCalculatedFieldDeleted(CalculatedField calculatedField, TbQueueCallback callback); + } diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java index b8f9fa4e5f..e15d9c8ace 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java @@ -15,8 +15,22 @@ */ package org.thingsboard.server.queue; + public interface TbQueueCallback { + TbQueueCallback EMPTY = new TbQueueCallback() { + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + + } + + @Override + public void onFailure(Throwable t) { + + } + }; + void onSuccess(TbQueueMsgMetadata metadata); void onFailure(Throwable t); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java index abde4cce97..609a61d575 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java @@ -26,6 +26,8 @@ public interface TbQueueRequestTemplate send(Request request, long timeoutNs); + ListenableFuture send(Request request, Integer partition); + void stop(); void setMessagesStats(MessagesStats messagesStats); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java index e7e5361381..918d656af0 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java @@ -15,9 +15,17 @@ */ package org.thingsboard.server.queue; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; + +import java.util.Set; + public interface TbQueueResponseTemplate { - void init(TbQueueHandler handler); + void subscribe(); + + void subscribe(Set partitions); + + void launch(TbQueueHandler handler); void stop(); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java index 7792d05a4c..7be61d15d5 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java @@ -118,6 +118,8 @@ public interface AlarmService extends EntityDaoService { long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query); + long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query, Collection orderedEntityIds); + PageData findAlarmTypesByTenantId(TenantId tenantId, PageLink pageLink); List findActiveOriginatorAlarms(TenantId tenantId, OriginatorAlarmFilter originatorAlarmFilter, int limit); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java index 04992800eb..09bc8f1f93 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.asset; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetSearchQuery; @@ -63,6 +64,10 @@ public interface AssetService extends EntityDaoService { PageData findAssetInfosByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink); + PageData findProfileEntityIdInfos(PageLink pageLink); + + PageData findAssetIdsByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink); + ListenableFuture> findAssetsByTenantIdAndIdsAsync(TenantId tenantId, List assetIds); void deleteAssetsByTenantId(TenantId tenantId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java new file mode 100644 index 0000000000..3d91790790 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.cf; + +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.entity.EntityDaoService; + +import java.util.List; + +public interface CalculatedFieldService extends EntityDaoService { + + CalculatedField save(CalculatedField calculatedField); + + CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId); + + List findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + + PageData findAllCalculatedFields(PageLink pageLink); + + PageData findAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink); + + void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + int deleteAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + + CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink); + + CalculatedFieldLink findCalculatedFieldLinkById(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId); + + List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + List findAllCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + + PageData findAllCalculatedFieldLinks(PageLink pageLink); + + boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); + +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java index bab59e6d66..4ef653855d 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.DeviceInfoFilter; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.device.DeviceSearchQuery; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; @@ -73,8 +74,12 @@ public interface DeviceService extends EntityDaoService { PageData findDeviceIdInfos(PageLink pageLink); + PageData findProfileEntityIdInfos(PageLink pageLink); + PageData findDevicesByTenantIdAndType(TenantId tenantId, String type, PageLink pageLink); + PageData findDeviceIdsByTenantIdAndDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId, PageLink pageLink); + PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, OtaPackageType type, PageLink pageLink); long countDevicesByTenantIdAndDeviceProfileIdAndEmptyOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, OtaPackageType otaPackageType); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java index d2fe51a133..65db0d5a76 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.entity; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.NameLabelAndCustomerDetails; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -34,6 +35,8 @@ public interface EntityService { Optional fetchEntityCustomerId(TenantId tenantId, EntityId entityId); + Optional> fetchEntity(TenantId tenantId, EntityId entityId); + Optional fetchNameLabelAndCustomerDetails(TenantId tenantId, EntityId entityId); long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java index d6074ca48b..ee187db46b 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.ResourceExportData; import org.thingsboard.server.common.data.ResourceSubType; import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TbResourceDeleteResult; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.TbResourceInfoFilter; import org.thingsboard.server.common.data.id.TbResourceId; @@ -69,9 +70,7 @@ public interface ResourceService extends EntityDaoService { PageData findTenantResourcesByResourceTypeAndPageLink(TenantId tenantId, ResourceType lwm2mModel, PageLink pageLink); - void deleteResource(TenantId tenantId, TbResourceId resourceId); - - void deleteResource(TenantId tenantId, TbResourceId resourceId, boolean force); + TbResourceDeleteResult deleteResource(TenantId tenantId, TbResourceId resourceId, boolean force); void deleteResourcesByTenantId(TenantId tenantId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java index b96a984f71..a2356ee149 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java @@ -46,6 +46,8 @@ public interface RuleChainService extends EntityDaoService { RuleChain saveRuleChain(RuleChain ruleChain, boolean publishSaveEvent); + RuleChain saveRuleChain(RuleChain ruleChain, boolean publishSaveEvent, boolean doValidate); + boolean setRootRuleChain(TenantId tenantId, RuleChainId ruleChainId); RuleChainUpdateResult saveRuleChainMetaData(TenantId tenantId, RuleChainMetaData ruleChainMetaData, Function ruleNodeUpdater); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java index 862c50e45e..e239e22ee9 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; @@ -44,13 +45,13 @@ public interface TimeseriesService { ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId); - ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); + ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); - ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); + ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); - ListenableFuture saveWithoutLatest(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); + ListenableFuture saveWithoutLatest(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); - ListenableFuture> saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntry); + ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntries); ListenableFuture> remove(TenantId tenantId, EntityId entityId, List queries); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java index e80a5d86e7..b53d6daec2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java @@ -145,4 +145,7 @@ public class DataConstants { public static final String EDGE_QUEUE_NAME = "Edge"; public static final String EDGE_EVENT_QUEUE_NAME = "EdgeEvent"; + public static final String CF_QUEUE_NAME = "CalculatedFields"; + public static final String CF_STATES_QUEUE_NAME = "CalculatedFieldStates"; + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index f5de438a9f..4ea66be1b4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -61,7 +61,9 @@ public enum EntityType { OAUTH2_CLIENT(35), DOMAIN(36), MOBILE_APP(37), - MOBILE_APP_BUNDLE(38); + MOBILE_APP_BUNDLE(38), + CALCULATED_FIELD(39), + CALCULATED_FIELD_LINK(40); @Getter private final int protoNumber; // Corresponds to EntityTypeProto diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java new file mode 100644 index 0000000000..51c631fe4f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java @@ -0,0 +1,89 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import java.util.EnumSet; +import java.util.List; +import java.util.Set; + +public enum ObjectType { + TENANT, + TENANT_PROFILE, + CUSTOMER, + QUEUE, + RPC, + RULE_CHAIN, + OTA_PACKAGE, + RESOURCE, + EVENT, + RULE_NODE, + USER, + EDGE, + WIDGETS_BUNDLE, + WIDGET_TYPE, + DASHBOARD, + DEVICE_PROFILE, + DEVICE, + DEVICE_CREDENTIALS, + ASSET_PROFILE, + ASSET, + ENTITY_VIEW, + ALARM, + ENTITY_ALARM, + OAUTH2_CLIENT, + OAUTH2_DOMAIN, + OAUTH2_MOBILE, + USER_SETTINGS, + NOTIFICATION_TARGET, + NOTIFICATION_TEMPLATE, + NOTIFICATION_RULE, + ALARM_COMMENT, + API_USAGE_STATE, + QUEUE_STATS, + + AUDIT_LOG, + RELATION, + ATTRIBUTE_KV, + LATEST_TS_KV; + + public static final Set edqsTenantTypes = EnumSet.of( + TENANT, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD, + RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, API_USAGE_STATE, QUEUE_STATS + ); + public static final Set edqsTypes = EnumSet.copyOf(edqsTenantTypes); + public static final Set edqsSystemTypes = EnumSet.of(TENANT, USER, DASHBOARD, + API_USAGE_STATE, ATTRIBUTE_KV, LATEST_TS_KV); + public static final Set unversionedTypes = EnumSet.of( + QUEUE_STATS // created once, never updated + ); + + static { + edqsTypes.addAll(List.of(RELATION, ATTRIBUTE_KV, LATEST_TS_KV)); + } + + public EntityType toEntityType() { + return EntityType.valueOf(name()); + } + + public static ObjectType fromEntityType(EntityType entityType) { + try { + return ObjectType.valueOf(entityType.name()); + } catch (Exception e) { + return null; + } + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java new file mode 100644 index 0000000000..22934de813 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.io.Serializable; +import java.util.UUID; + +@Data +@Slf4j +public class ProfileEntityIdInfo implements Serializable, HasTenantId { + + private static final long serialVersionUID = 8532058281983868003L; + + private final TenantId tenantId; + private final EntityId profileId; + private final EntityId entityId; + + private ProfileEntityIdInfo(UUID tenantId, EntityId profileId, EntityId entityId) { + this.tenantId = TenantId.fromUUID(tenantId); + this.profileId = profileId; + this.entityId = entityId; + } + + public static ProfileEntityIdInfo create(UUID tenantId, DeviceProfileId profileId, DeviceId entityId) { + return new ProfileEntityIdInfo(tenantId, profileId, entityId); + } + + public static ProfileEntityIdInfo create(UUID tenantId, AssetProfileId profileId, AssetId entityId) { + return new ProfileEntityIdInfo(tenantId, profileId, entityId); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java index 0856f65264..b1ef4d7f22 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java @@ -34,4 +34,7 @@ public class SystemParams { boolean mobileQrEnabled; int maxDebugModeDurationMinutes; String ruleChainDebugPerTenantLimitsConfiguration; + String calculatedFieldDebugPerTenantLimitsConfiguration; + long maxArgumentsPerCF; + long maxDataPointsPerRollingArg; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TbResourceDeleteResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/TbResourceDeleteResult.java new file mode 100644 index 0000000000..edc5a2f539 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/TbResourceDeleteResult.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.id.HasId; + +import java.util.List; +import java.util.Map; + +@Data +@Builder +public class TbResourceDeleteResult { + + private boolean success; + private Map>> references; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java index b7c7584931..5c7f0bb53d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.Valid; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.ToString; @@ -58,6 +59,7 @@ public class TenantProfile extends BaseData implements HasName @Schema(description = "If enabled, will push all messages related to this tenant and processed by the rule engine into separate queue. " + "Useful for complex microservices deployments, to isolate processing of the data for specific tenants", example = "false") private boolean isolatedTbRuleEngine; + @Valid @Schema(description = "Complex JSON object that contains profile settings: queue configs, max devices, max assets, rate limits, etc.") private transient TenantProfileData profileData; @JsonIgnore diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java new file mode 100644 index 0000000000..b86f30ca78 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -0,0 +1,143 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSetter; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.HasDebugSettings; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.HasVersion; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.validation.Length; +import org.thingsboard.server.common.data.validation.NoXss; + +import java.io.Serial; + +@Schema +@Data +@EqualsAndHashCode(callSuper = true) +public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion, HasDebugSettings { + + @Serial + private static final long serialVersionUID = 4491966747773381420L; + + private TenantId tenantId; + private EntityId entityId; + + @NoXss + @Length(fieldName = "type") + private CalculatedFieldType type; + @NoXss + @Length(fieldName = "name") + @Schema(description = "User defined name of the calculated field.") + private String name; + @Deprecated + @Schema(description = "Enable/disable debug. ", example = "false", deprecated = true) + private boolean debugMode; + @Schema(description = "Debug settings object.") + private DebugSettings debugSettings; + @Schema(description = "Version of calculated field configuration.", example = "0") + private int configurationVersion; + @Schema(implementation = SimpleCalculatedFieldConfiguration.class) + private transient CalculatedFieldConfiguration configuration; + @Getter + @Setter + private Long version; + + public CalculatedField() { + super(); + } + + public CalculatedField(CalculatedFieldId id) { + super(id); + } + + public CalculatedField(TenantId tenantId, EntityId entityId, CalculatedFieldType type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version) { + this.tenantId = tenantId; + this.entityId = entityId; + this.type = type; + this.name = name; + this.configurationVersion = configurationVersion; + this.configuration = configuration; + this.version = version; + } + + public CalculatedField(CalculatedField calculatedField) { + super(calculatedField); + this.tenantId = calculatedField.tenantId; + this.entityId = calculatedField.entityId; + this.type = calculatedField.type; + this.name = calculatedField.name; + this.debugMode = calculatedField.debugMode; + this.debugSettings = calculatedField.debugSettings; + this.configurationVersion = calculatedField.configurationVersion; + this.configuration = calculatedField.configuration; + this.version = calculatedField.version; + } + + @Schema(description = "JSON object with the Calculated Field Id. Referencing non-existing Calculated Field Id will cause error.") + @Override + public CalculatedFieldId getId() { + return super.getId(); + } + + @Schema(description = "Timestamp of the calculated field creation, in milliseconds", example = "1609459200000", accessMode = Schema.AccessMode.READ_ONLY) + @Override + public long getCreatedTime() { + return super.getCreatedTime(); + } + + @Override + public String toString() { + return new StringBuilder() + .append("CalculatedField[") + .append("tenantId=").append(tenantId) + .append(", entityId=").append(entityId) + .append(", type='").append(type) + .append(", name='").append(name) + .append(", configurationVersion=").append(configurationVersion) + .append(", configuration=").append(configuration) + .append(", version=").append(version) + .append(", createdTime=").append(createdTime) + .append(", id=").append(id).append(']') + .toString(); + } + + // Getter is ignored for serialization + @JsonIgnore + public boolean isDebugMode() { + return debugMode; + } + + // Setter is annotated for deserialization + @JsonSetter + public void setDebugMode(boolean debugMode) { + this.debugMode = debugMode; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java new file mode 100644 index 0000000000..3f048815da --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java @@ -0,0 +1,66 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +@Schema +@Data +@EqualsAndHashCode(callSuper = true) +public class CalculatedFieldLink extends BaseData { + + private static final long serialVersionUID = 6492846246722091530L; + + private TenantId tenantId; + private EntityId entityId; + + @Schema(description = "JSON object with the Calculated Field Id. ", accessMode = Schema.AccessMode.READ_ONLY) + private CalculatedFieldId calculatedFieldId; + + public CalculatedFieldLink() { + super(); + } + + public CalculatedFieldLink(CalculatedFieldLinkId id) { + super(id); + } + + public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId) { + this.tenantId = tenantId; + this.entityId = entityId; + this.calculatedFieldId = calculatedFieldId; + } + + @Override + public String toString() { + return new StringBuilder() + .append("CalculatedFieldLink[") + .append("tenantId=").append(tenantId) + .append(", entityId=").append(entityId) + .append(", calculatedFieldId=").append(calculatedFieldId) + .append(", createdTime=").append(createdTime) + .append(", id=").append(id).append(']') + .toString(); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java new file mode 100644 index 0000000000..acef67a041 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +public enum CalculatedFieldType { + + SIMPLE, SCRIPT + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java new file mode 100644 index 0000000000..e7daa70b1b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Data; +import org.springframework.lang.Nullable; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Argument { + + @Nullable + private EntityId refEntityId; + private ReferencedEntityKey refEntityKey; + private String defaultValue; + + private Integer limit; + private Long timeWindow; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java new file mode 100644 index 0000000000..7f057f4038 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +public enum ArgumentType { + + TS_LATEST, ATTRIBUTE, TS_ROLLING + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..8227ff4603 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +@Data +public abstract class BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + protected Map arguments; + protected String expression; + protected Output output; + + @Override + public List getReferencedEntities() { + return arguments.values().stream() + .map(Argument::getRefEntityId) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + @Override + public List buildCalculatedFieldLinks(TenantId tenantId, EntityId cfEntityId, CalculatedFieldId calculatedFieldId) { + return getReferencedEntities().stream() + .filter(referencedEntity -> !referencedEntity.equals(cfEntityId)) + .map(referencedEntityId -> buildCalculatedFieldLink(tenantId, referencedEntityId, calculatedFieldId)) + .collect(Collectors.toList()); + } + + @Override + public CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, EntityId referencedEntityId, CalculatedFieldId calculatedFieldId) { + CalculatedFieldLink link = new CalculatedFieldLink(); + link.setTenantId(tenantId); + link.setEntityId(referencedEntityId); + link.setCalculatedFieldId(calculatedFieldId); + return link; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java new file mode 100644 index 0000000000..c53f1fe5f1 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.List; +import java.util.Map; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE"), + @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT") +}) +public interface CalculatedFieldConfiguration { + + @JsonIgnore + CalculatedFieldType getType(); + + Map getArguments(); + + String getExpression(); + + void setExpression(String expression); + + Output getOutput(); + + @JsonIgnore + List getReferencedEntities(); + + List buildCalculatedFieldLinks(TenantId tenantId, EntityId cfEntityId, CalculatedFieldId calculatedFieldId); + + CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, EntityId referencedEntityId, CalculatedFieldId calculatedFieldId); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java new file mode 100644 index 0000000000..f2b4948837 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; + +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Output { + + private String name; + private OutputType type; + private AttributeScope scope; + private Integer decimalsByDefault; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java new file mode 100644 index 0000000000..04a816b74f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +public enum OutputType { + + TIME_SERIES, ATTRIBUTES + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java new file mode 100644 index 0000000000..9e3a75c891 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; + +@Data +@AllArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ReferencedEntityKey { + + private String key; + private ArgumentType type; + private AttributeScope scope; + +} diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNodeConfigurationTest.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java similarity index 60% rename from rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNodeConfigurationTest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java index 5c4d341fda..0971217fdf 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNodeConfigurationTest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java @@ -13,17 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.rule.engine.telemetry; +package org.thingsboard.server.common.data.cf.configuration; -import org.junit.jupiter.api.Test; +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import static org.assertj.core.api.Assertions.assertThat; +@Data +public class ScriptCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { -class TbMsgAttributesNodeConfigurationTest { - - @Test - void testDefaultConfig_givenUpdateAttributesOnlyOnValueChange_thenTrue_sinceVersion1() { - assertThat(new TbMsgAttributesNodeConfiguration().defaultConfiguration().isUpdateAttributesOnlyOnValueChange()).isTrue(); + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SCRIPT; } - } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..79a0518ba0 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; + +@Data +public class SimpleCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SIMPLE; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java new file mode 100644 index 0000000000..c162365257 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java @@ -0,0 +1,75 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class AttributeKv implements EdqsObject { + + private EntityId entityId; + private AttributeScope scope; + private String key; + private Long version; + + private DataPoint dataPoint; // optional (on deletion) + + private Long lastUpdateTs; // only for serialization + private KvEntry value; // only for serialization + + public AttributeKv(EntityId entityId, AttributeScope scope, AttributeKvEntry attributeKvEntry, long version) { + this.entityId = entityId; + this.scope = scope; + this.key = attributeKvEntry.getKey(); + this.version = version; + this.lastUpdateTs = attributeKvEntry.getLastUpdateTs(); + this.value = attributeKvEntry; + } + + public AttributeKv(EntityId entityId, AttributeScope scope, String key, long version) { + this.entityId = entityId; + this.scope = scope; + this.key = key; + this.version = version; + } + + @Override + public String key() { + return "a_" + entityId + "_" + scope + "_" + key; + } + + @Override + public Long version() { + return version; + } + + @Override + public ObjectType type() { + return ObjectType.ATTRIBUTE_KV; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/DataPoint.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/DataPoint.java new file mode 100644 index 0000000000..a6f30c8004 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/DataPoint.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import org.thingsboard.server.common.data.kv.DataType; + +public interface DataPoint { + + String NOT_SUPPORTED = "Not supported!"; + + long getTs(); + + DataType getType(); + + String getStr(); + + long getLong(); + + double getDouble(); + + boolean getBool(); + + String getJson(); + + String valueToString(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEvent.java new file mode 100644 index 0000000000..50c8d268de --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEvent.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +@AllArgsConstructor +@Builder +public class EdqsEvent { + + private final TenantId tenantId; + private final ObjectType objectType; + private final EdqsEventType eventType; + private final EdqsObject object; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEventType.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEventType.java new file mode 100644 index 0000000000..df31048365 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEventType.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +public enum EdqsEventType { + UPDATED, + DELETED +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java new file mode 100644 index 0000000000..a74c90208a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import org.thingsboard.server.common.data.ObjectType; + +public interface EdqsObject { + + @JsonIgnore + String key(); + + @JsonIgnore + Long version(); + + @JsonIgnore + ObjectType type(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsSyncRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsSyncRequest.java new file mode 100644 index 0000000000..12f7068c71 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsSyncRequest.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Data; + +@Data +@JsonIgnoreProperties +public class EdqsSyncRequest { +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java new file mode 100644 index 0000000000..c22ef147e3 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java @@ -0,0 +1,68 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; +import org.thingsboard.server.common.data.edqs.fields.EntityIdFields; + +import java.util.UUID; + +@Data +@NoArgsConstructor +public class Entity implements EdqsObject { + + private EntityType type; + + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) + private EntityFields fields; + + public Entity(EntityType type) { + this.type = type; + } + + public Entity(EntityType type, EntityFields fields) { + this.type = type; + this.fields = fields; + } + + public Entity(EntityType entityType, UUID id, long version) { + this.type = entityType; + this.fields = new EntityIdFields(id, version); + } + + @Override + public String key() { + return "e_" + fields.getId().toString(); + } + + @Override + public Long version() { + return fields.getVersion(); + } + + @Override + public ObjectType type() { + return ObjectType.fromEntityType(type); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java new file mode 100644 index 0000000000..8bd69c41a4 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java @@ -0,0 +1,70 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class LatestTsKv implements EdqsObject { + + private EntityId entityId; + private String key; + private Long version; + + private DataPoint dataPoint; // optional (on deletion) + + private Long ts; // only for serialization + private KvEntry value; // only for serialization + + public LatestTsKv(EntityId entityId, TsKvEntry tsKvEntry, Long version) { + this.entityId = entityId; + this.key = tsKvEntry.getKey(); + this.ts = tsKvEntry.getTs(); + this.version = version != null ? version : 0L; + this.value = tsKvEntry; + } + + public LatestTsKv(EntityId entityId, String key, Long version) { + this.entityId = entityId; + this.key = key; + this.version = version != null ? version : 0L; + } + + public String key() { + return "l_" + entityId + "_" + key; + } + + @Override + public Long version() { + return version; + } + + @Override + public ObjectType type() { + return ObjectType.LATEST_TS_KV; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsMsg.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsMsg.java new file mode 100644 index 0000000000..78bebba20a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsMsg.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class ToCoreEdqsMsg { + + private EdqsSyncRequest syncRequest; + private Boolean apiEnabled; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsRequest.java new file mode 100644 index 0000000000..c4f262fbf0 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsRequest.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class ToCoreEdqsRequest { + + private EdqsSyncRequest syncRequest; + private Boolean apiEnabled; + + @JsonIgnore + public ToCoreEdqsMsg toInternalMsg() { + return new ToCoreEdqsMsg(syncRequest, apiEnabled); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AbstractEntityFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AbstractEntityFields.java new file mode 100644 index 0000000000..4aad5eb4dd --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AbstractEntityFields.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.experimental.SuperBuilder; +import org.thingsboard.server.common.data.id.CustomerId; + +import java.util.UUID; + +@Data +@SuperBuilder +public class AbstractEntityFields implements EntityFields { + + private UUID id; + private long createdTime; + private UUID tenantId; + private UUID customerId; + private String name; + private Long version; + + public AbstractEntityFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, Long version) { + this.id = id; + this.createdTime = createdTime; + this.tenantId = tenantId; + this.customerId = (customerId != null && customerId != CustomerId.NULL_UUID) ? customerId : null; + this.name = name; + this.version = version; + } + + public AbstractEntityFields() { + } + + public AbstractEntityFields(UUID id, long createdTime, UUID tenantId, String name, Long version) { + this(id, createdTime, tenantId, null, name, version); + } + + public AbstractEntityFields(UUID id, long createdTime, UUID tenantId, UUID customerId, Long version) { + this(id, createdTime, tenantId, customerId, null, version); + + } + + public AbstractEntityFields(UUID id, long createdTime, String name, Long version) { + this(id, createdTime, null, name, version); + } + + + public AbstractEntityFields(UUID id, long createdTime, UUID tenantId) { + this(id, createdTime, tenantId, null, null, null); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java new file mode 100644 index 0000000000..d10f375bc1 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; +import org.thingsboard.server.common.data.ApiUsageStateValue; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; + +import java.util.UUID; + +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor +@SuperBuilder +public class ApiUsageStateFields extends AbstractEntityFields { + + private EntityId entityId; + private ApiUsageStateValue transportState; + private ApiUsageStateValue dbStorageState; + private ApiUsageStateValue reExecState; + private ApiUsageStateValue jsExecState; + private ApiUsageStateValue tbelExecState; + private ApiUsageStateValue emailExecState; + private ApiUsageStateValue smsExecState; + private ApiUsageStateValue alarmExecState; + + public ApiUsageStateFields(UUID id, long createdTime, UUID tenantId, UUID entityId, String entityType, ApiUsageStateValue transportState, ApiUsageStateValue dbStorageState, + ApiUsageStateValue reExecState, ApiUsageStateValue jsExecState, ApiUsageStateValue tbelExecState, + ApiUsageStateValue emailExecState, ApiUsageStateValue smsExecState, ApiUsageStateValue alarmExecState, + Long version) { + super(id, createdTime, tenantId, null, null, version); + this.entityId = (entityType != null && entityId != null) ? EntityIdFactory.getByTypeAndUuid(entityType, entityId) : null; + this.transportState = transportState; + this.dbStorageState = dbStorageState; + this.reExecState = reExecState; + this.jsExecState = jsExecState; + this.tbelExecState = tbelExecState; + this.emailExecState = emailExecState; + this.smsExecState = smsExecState; + this.alarmExecState = alarmExecState; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetFields.java new file mode 100644 index 0000000000..b4020e5788 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetFields.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class AssetFields extends AbstractEntityFields implements ProfileAwareFields { + + private String type; + private UUID assetProfileId; + private String label; + private String additionalInfo; + + @JsonIgnore + @Override + public String getProfileName() { + return type; + } + + @JsonIgnore + @Override + public UUID getProfileId() { + return assetProfileId; + } + + public AssetFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, + Long version, String type, String label, UUID assetProfileId, JsonNode additionalInfo) { + super(id, createdTime, tenantId, customerId, name, version); + this.type = type; + this.assetProfileId = assetProfileId; + this.label = label; + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetProfileFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetProfileFields.java new file mode 100644 index 0000000000..56d8691cbe --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetProfileFields.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +@Data +@NoArgsConstructor +@SuperBuilder +public class AssetProfileFields extends AbstractEntityFields { + + private boolean isDefault; + + public AssetProfileFields(UUID id, long createdTime, UUID tenantId, String name, Long version, boolean isDefault) { + super(id, createdTime, tenantId, null, name, version); + this.isDefault = isDefault; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/CustomerFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/CustomerFields.java new file mode 100644 index 0000000000..4132e9e094 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/CustomerFields.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class CustomerFields extends AbstractEntityFields { + + private String additionalInfo; + private String country; + private String state; + private String city; + private String address; + private String address2; + private String zip; + private String phone; + private String email; + + public CustomerFields(UUID id, long createdTime, UUID tenantId, String name, Long version, JsonNode additionalInfo, + String country, String state, String city, String address, String address2, String zip, String phone, String email) { + super(id, createdTime, tenantId, name, version); + this.additionalInfo = getText(additionalInfo); + this.country = country; + this.state = state; + this.city = city; + this.address = address; + this.address2 = address2; + this.zip = zip; + this.phone = phone; + this.email = email; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java new file mode 100644 index 0000000000..0061b7e10b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + + +@Data +@NoArgsConstructor +@SuperBuilder +public class DashboardFields extends AbstractEntityFields { + + private static ObjectMapper objectMapper = new ObjectMapper(); + private List assignedCustomerIds; + + public DashboardFields(UUID id, long createdTime, UUID tenantId, String assignedCustomers, String name, Long version) { + super(id, createdTime, tenantId, name, version); + this.assignedCustomerIds = getCustomerIds(assignedCustomers); + } + + private static List getCustomerIds(String assignedCustomers) { + List ids = new ArrayList<>(); + if (assignedCustomers == null || assignedCustomers.isEmpty()) { + return ids; + } + try { + JsonNode rootNode = objectMapper.readTree(assignedCustomers); + for (JsonNode node : rootNode) { + String idStr = node.path("customerId").path("id").asText(); + if (!idStr.isEmpty()) { + ids.add(UUID.fromString(idStr)); + } + } + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + return ids; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceFields.java new file mode 100644 index 0000000000..ea1ef383de --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceFields.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class DeviceFields extends AbstractEntityFields implements ProfileAwareFields { + + private String label; + private String type; + private UUID deviceProfileId; + private String additionalInfo; + + @JsonIgnore + @Override + public String getProfileName() { + return type; + } + + @JsonIgnore + @Override + public UUID getProfileId() { + return deviceProfileId; + } + + public DeviceFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, Long version, String type, + String label, UUID deviceProfileId, JsonNode additionalInfo) { + super(id, createdTime, tenantId, customerId, name, version); + this.label = label; + this.type = type; + this.deviceProfileId = deviceProfileId; + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceProfileFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceProfileFields.java new file mode 100644 index 0000000000..1e8d56ab14 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceProfileFields.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; +import org.thingsboard.server.common.data.DeviceProfileType; + +import java.util.UUID; + +@Data +@NoArgsConstructor +@SuperBuilder +public class DeviceProfileFields extends AbstractEntityFields { + + private String type; + private boolean isDefault; + + public DeviceProfileFields(UUID id, long createdTime, UUID tenantId, String name, Long version, DeviceProfileType type, boolean isDefault) { + super(id, createdTime, tenantId, null, name, version); + this.type = type.name(); + this.isDefault = isDefault; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EdgeFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EdgeFields.java new file mode 100644 index 0000000000..483a7f2680 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EdgeFields.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class EdgeFields extends AbstractEntityFields { + + private String type; + private String label; + private String additionalInfo; + + public EdgeFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, Long version, + String type, String label, JsonNode additionalInfo) { + super(id, createdTime, tenantId, customerId, name, version); + this.type = type; + this.label = label; + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java new file mode 100644 index 0000000000..532c4a92ac --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java @@ -0,0 +1,179 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.annotation.JsonInclude; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public interface EntityFields { + + Logger log = LoggerFactory.getLogger(EntityFields.class); + + default UUID getId() { + return null; + } + + default UUID getTenantId() { + return null; + } + + default UUID getCustomerId() { + return null; + } + + default List getAssignedCustomerIds() { + return Collections.emptyList(); + } + + default long getCreatedTime() { + return 0; + } + + default String getName() { + return ""; + } + + default String getType() { + return ""; + } + + default String getLabel() { + return ""; + } + + default String getAdditionalInfo() { + return ""; + } + + default String getEmail() { + return ""; + } + + default String getCountry() { + return ""; + } + + default String getState() { + return ""; + } + + default String getCity() { + return ""; + } + + default String getAddress() { + return ""; + } + + default String getAddress2() { + return ""; + } + + default String getZip() { + return ""; + } + + default String getPhone() { + return ""; + } + + default String getRegion() { + return ""; + } + + default String getFirstName() { + return ""; + } + + default String getLastName() { + return ""; + } + + default boolean isEdgeTemplate() { + return false; + } + + default String getConfiguration() { + return ""; + } + + default String getSchedule() { + return ""; + } + + default EntityId getOriginatorId() { + return null; + } + + default String getQueueName() { + return ""; + } + + default String getServiceId() { + return ""; + } + + default boolean isDefault() { + return false; + } + + default UUID getOwnerId() { + return null; + } + + default Long getVersion() { + return null; + } + + default String getAsString(String key) { + return switch (key) { + case "createdTime" -> Long.toString(getCreatedTime()); + case "type" -> getType(); + case "label" -> getLabel(); + case "additionalInfo" -> getAdditionalInfo(); + case "email" -> getEmail(); + case "country" -> getCountry(); + case "state" -> getState(); + case "city" -> getCity(); + case "address" -> getAddress(); + case "address2" -> getAddress2(); + case "zip" -> getZip(); + case "phone" -> getPhone(); + case "region" -> getRegion(); + case "firstName" -> getFirstName(); + case "lastName" -> getLastName(); + case "edgeTemplate" -> Boolean.toString(isEdgeTemplate()); + case "configuration" -> getConfiguration(); + case "schedule" -> getSchedule(); + case "originatorId" -> getOriginatorId().getId().toString(); + case "originatorType" -> getOriginatorId().getEntityType().toString(); + case "queueName" -> getQueueName(); + case "serviceId" -> getServiceId(); + default -> { + log.warn("Unknown field '{}'", key); + yield null; + } + }; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityIdFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityIdFields.java new file mode 100644 index 0000000000..835577c4f1 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityIdFields.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +@Data +@NoArgsConstructor +@SuperBuilder +public class EntityIdFields implements EntityFields { + + private UUID id; + private Long version; + + public EntityIdFields(UUID id, Long version) { + this.id = id; + this.version = version; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java new file mode 100644 index 0000000000..ba3c105a87 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class EntityViewFields extends AbstractEntityFields { + + private String type; + private String additionalInfo; + + public EntityViewFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, String type, JsonNode additionalInfo, Long version) { + super(id, createdTime, tenantId, customerId, name, version); + this.type = type; + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java new file mode 100644 index 0000000000..a36514248c --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java @@ -0,0 +1,299 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.queue.QueueStats; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.common.data.widget.WidgetType; +import org.thingsboard.server.common.data.widget.WidgetsBundle; + +import java.util.UUID; + +public class FieldsUtil { + + public static EntityFields toFields(Object entity) { + if (entity instanceof Customer customer) { + return toFields(customer); + } else if (entity instanceof Tenant tenant) { + return toFields(tenant); + } else if (entity instanceof TenantProfile tenantProfile) { + return toFields(tenantProfile); + } else if (entity instanceof Device device) { + return toFields(device); + } else if (entity instanceof Asset asset) { + return toFields(asset); + } else if (entity instanceof Edge edge) { + return toFields(edge); + } else if (entity instanceof EntityView entityView) { + return toFields(entityView); + } else if (entity instanceof User user) { + return toFields(user); + } else if (entity instanceof Dashboard dashboard) { + return toFields(dashboard); + } else if (entity instanceof RuleChain ruleChain) { + return toFields(ruleChain); + } else if (entity instanceof RuleNode ruleNode) { + return toFields(ruleNode); + } else if (entity instanceof WidgetType widgetType) { + return toFields(widgetType); + } else if (entity instanceof WidgetsBundle widgetsBundle) { + return toFields(widgetsBundle); + } else if (entity instanceof DeviceProfile deviceProfile) { + return toFields(deviceProfile); + } else if (entity instanceof AssetProfile assetProfile) { + return toFields(assetProfile); + } else if (entity instanceof QueueStats queueStats) { + return toFields(queueStats); + } else if (entity instanceof ApiUsageState apiUsageState) { + return toFields(apiUsageState); + } else { + throw new IllegalArgumentException("Unsupported entity type: " + entity.getClass().getName()); + } + } + + private static CustomerFields toFields(Customer entity) { + return CustomerFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getTitle()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .email(entity.getEmail()) + .country(entity.getCountry()) + .state(entity.getState()) + .city(entity.getCity()) + .address(entity.getAddress()) + .address2(entity.getAddress2()) + .zip(entity.getZip()) + .phone(entity.getPhone()) + .version(entity.getVersion()) + .build(); + } + + private static TenantFields toFields(Tenant entity) { + return TenantFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getTitle()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .email(entity.getEmail()) + .country(entity.getCountry()) + .state(entity.getState()) + .city(entity.getCity()) + .address(entity.getAddress()) + .address2(entity.getAddress2()) + .zip(entity.getZip()) + .phone(entity.getPhone()) + .region(entity.getRegion()) + .version(entity.getVersion()) + .build(); + } + + private static TenantProfileFields toFields(TenantProfile tenantProfile) { + return TenantProfileFields.builder() + .id(tenantProfile.getUuidId()) + .createdTime(tenantProfile.getCreatedTime()) + .name(tenantProfile.getName()) + .isDefault(tenantProfile.isDefault()) + .build(); + } + + private static DeviceFields toFields(Device entity) { + return DeviceFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .name(entity.getName()) + .type(entity.getType()) + .deviceProfileId(entity.getDeviceProfileId().getId()) + .label(entity.getLabel()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static AssetFields toFields(Asset entity) { + return AssetFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .name(entity.getName()) + .type(entity.getType()) + .assetProfileId(entity.getAssetProfileId().getId()) + .label(entity.getLabel()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static EdgeFields toFields(Edge entity) { + return EdgeFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .name(entity.getName()) + .type(entity.getType()) + .label(entity.getLabel()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static EntityViewFields toFields(EntityView entity) { + return EntityViewFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .name(entity.getName()) + .type(entity.getType()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static UserFields toFields(User entity) { + return UserFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .firstName(entity.getFirstName()) + .lastName(entity.getLastName()) + .email(entity.getEmail()) + .phone(entity.getPhone()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static DashboardFields toFields(Dashboard entity) { + return DashboardFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getTitle()) + .version(entity.getVersion()) + .build(); + } + + private static RuleChainFields toFields(RuleChain entity) { + return RuleChainFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static RuleNodeFields toFields(RuleNode entity) { + return RuleNodeFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .build(); + } + + private static WidgetTypeFields toFields(WidgetType entity) { + return WidgetTypeFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .version(entity.getVersion()) + .build(); + } + + private static WidgetsBundleFields toFields(WidgetsBundle entity) { + return WidgetsBundleFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .version(entity.getVersion()) + .build(); + } + + private static AssetProfileFields toFields(AssetProfile entity) { + return AssetProfileFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .isDefault(entity.isDefault()) + .version(entity.getVersion()) + .build(); + } + + private static DeviceProfileFields toFields(DeviceProfile entity) { + return DeviceProfileFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .type(DeviceProfileType.DEFAULT.name()) + .isDefault(entity.isDefault()) + .version(entity.getVersion()) + .build(); + } + + private static QueueStatsFields toFields(QueueStats entity) { + return QueueStatsFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .queueName(entity.getQueueName()) + .serviceId(entity.getServiceId()) + .build(); + } + + private static ApiUsageStateFields toFields(ApiUsageState entity) { + return ApiUsageStateFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(entity.getEntityId().getEntityType() == EntityType.CUSTOMER ? entity.getEntityId().getId() : null) + .entityId(entity.getEntityId()) + .transportState(entity.getTransportState()) + .dbStorageState(entity.getDbStorageState()) + .reExecState(entity.getReExecState()) + .jsExecState(entity.getJsExecState()) + .tbelExecState(entity.getTbelExecState()) + .emailExecState(entity.getEmailExecState()) + .smsExecState(entity.getSmsExecState()) + .alarmExecState(entity.getAlarmExecState()) + .version(entity.getVersion()) + .build(); + } + + public static String getText(JsonNode node) { + return node != null ? node.asText() : ""; + } + + private static UUID getCustomerId(CustomerId customerId) { + return (customerId != null && !customerId.getId().equals(CustomerId.NULL_UUID)) ? customerId.getId() : null; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/GenericFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/GenericFields.java new file mode 100644 index 0000000000..68366a6b4d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/GenericFields.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.NoArgsConstructor; + +import java.util.UUID; + +@NoArgsConstructor +public class GenericFields extends AbstractEntityFields { + + public GenericFields(UUID id, long createdTime, UUID tenantId, String name, Long version) { + super(id, createdTime, tenantId, name, version); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ProfileAwareFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ProfileAwareFields.java new file mode 100644 index 0000000000..4228755808 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ProfileAwareFields.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import java.util.UUID; + +public interface ProfileAwareFields extends EntityFields { + + String getProfileName(); + + UUID getProfileId(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/QueueStatsFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/QueueStatsFields.java new file mode 100644 index 0000000000..cbc1ad4b8e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/QueueStatsFields.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +@Data +@NoArgsConstructor +@SuperBuilder +public class QueueStatsFields extends AbstractEntityFields { + + private String queueName; + private String serviceId; + + @Override + public String getName() { + return queueName + '_' + serviceId; + } + + public QueueStatsFields(UUID id, long createdTime, UUID tenantId, String queueName, String serviceId) { + super(id, createdTime, tenantId); + this.queueName = queueName; + this.serviceId = serviceId; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleChainFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleChainFields.java new file mode 100644 index 0000000000..a047eebd85 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleChainFields.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class RuleChainFields extends AbstractEntityFields { + + private String additionalInfo; + + public RuleChainFields(UUID id, long createdTime, UUID tenantId, String name, Long version, JsonNode additionalInfo) { + super(id, createdTime, tenantId, name, version); + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleNodeFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleNodeFields.java new file mode 100644 index 0000000000..8aa5b7f42f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleNodeFields.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class RuleNodeFields implements EntityFields { + + private UUID id; + private long createdTime; + private String name; + private String additionalInfo; + + public RuleNodeFields(UUID id, long createdTime, String name, JsonNode additionalInfo) { + this.id = id; + this.createdTime = createdTime; + this.name = name; + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java new file mode 100644 index 0000000000..342e2974a4 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class TenantFields extends AbstractEntityFields { + + private String additionalInfo; + private String country; + private String state; + private String city; + private String address; + private String address2; + private String zip; + private String phone; + private String email; + private String region; + + public TenantFields(UUID id, long createdTime, String name, Long version, + JsonNode additionalInfo, String country, String state, String city, String address, + String address2, String zip, String phone, String email, String region) { + super(id, createdTime, name, version); + this.additionalInfo = getText(additionalInfo); + this.country = country; + this.state = state; + this.city = city; + this.address = address; + this.address2 = address2; + this.zip = zip; + this.phone = phone; + this.email = email; + this.region = region; + } + + @Override + public UUID getTenantId() { + return getId(); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantProfileFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantProfileFields.java new file mode 100644 index 0000000000..b897bd1334 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantProfileFields.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; + +@Data +@NoArgsConstructor +@SuperBuilder +public class TenantProfileFields extends AbstractEntityFields { + + private boolean isDefault; + + public TenantProfileFields(UUID id, long createdTime, String name, boolean isDefault) { + super(id, createdTime, TenantId.SYS_TENANT_ID.getId(), null, name, 0L); + this.isDefault = isDefault; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/UserFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/UserFields.java new file mode 100644 index 0000000000..9863506ed4 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/UserFields.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class UserFields extends AbstractEntityFields { + + private String firstName; + private String lastName; + private String email; + private String phone; + private String additionalInfo; + + public UserFields(UUID id, long createdTime, UUID tenantId, UUID customerId, + Long version, String firstName, String lastName, String email, + String phone, JsonNode additionalInfo) { + super(id, createdTime, tenantId, customerId, version); + this.firstName = firstName; + this.lastName = lastName; + this.email = email; + this.phone = phone; + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetTypeFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetTypeFields.java new file mode 100644 index 0000000000..4fd0079012 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetTypeFields.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +@NoArgsConstructor +@SuperBuilder +public class WidgetTypeFields extends AbstractEntityFields { + + public WidgetTypeFields(UUID id, long createdTime, UUID tenantId, String name, Long version) { + super(id, createdTime, tenantId, name, version); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetsBundleFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetsBundleFields.java new file mode 100644 index 0000000000..f2fb1df508 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetsBundleFields.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +@NoArgsConstructor +@SuperBuilder +public class WidgetsBundleFields extends AbstractEntityFields { + + public WidgetsBundleFields(UUID id, long createdTime, UUID tenantId, String name, Long version) { + super(id, createdTime, tenantId, name, version); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java new file mode 100644 index 0000000000..e7d8b1df49 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.query; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EdqsRequest { + + private EntityDataQuery entityDataQuery; + private EntityCountQuery entityCountQuery; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsResponse.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsResponse.java new file mode 100644 index 0000000000..d4e53dea9f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsResponse.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.query; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityData; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@JsonIgnoreProperties(ignoreUnknown = true) +public class EdqsResponse { + + private PageData entityDataQueryResult; + private Long entityCountQueryResult; + private String error; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java new file mode 100644 index 0000000000..67a05323a1 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.query; + +import lombok.Data; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.TsValue; + +import java.util.Collections; +import java.util.Map; + +@Data +@RequiredArgsConstructor +public class QueryResult { + + private final EntityId entityId; + private final Map> latest; + + public EntityData toOldEntityData() { + return new EntityData(entityId, latest, Collections.emptyMap(), Collections.emptyMap()); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java new file mode 100644 index 0000000000..0424eabeb6 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java @@ -0,0 +1,94 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; + +@ToString +@EqualsAndHashCode(callSuper = true) +public class CalculatedFieldDebugEvent extends Event { + + private static final long serialVersionUID = -7091690784759639853L; + + @Builder + private CalculatedFieldDebugEvent(TenantId tenantId, UUID entityId, String serviceId, UUID id, long ts, + CalculatedFieldId calculatedFieldId, EntityId eventEntity, UUID msgId, + String msgType, String arguments, String result, String error) { + super(tenantId, entityId, serviceId, id, ts); + this.calculatedFieldId = calculatedFieldId; + this.eventEntity = eventEntity; + this.msgId = msgId; + this.msgType = msgType; + this.arguments = arguments; + this.result = result; + this.error = error; + } + + @Getter + private final CalculatedFieldId calculatedFieldId; + @Getter + private final EntityId eventEntity; + @Getter + private final UUID msgId; + @Getter + private final String msgType; + @Getter + @Setter + private String arguments; + @Getter + @Setter + private String result; + @Getter + @Setter + private String error; + + @Override + public EventType getType() { + return EventType.DEBUG_CALCULATED_FIELD; + } + + @Override + public EventInfo toInfo(EntityType entityType) { + EventInfo eventInfo = super.toInfo(entityType); + var json = (ObjectNode) eventInfo.getBody(); + json.put("calculatedFieldId", calculatedFieldId.toString()); + if (eventEntity != null) { + json.put("entityId", eventEntity.getId().toString()) + .put("entityType", eventEntity.getEntityType().name()); + } + if (msgId != null) { + json.put("msgId", msgId.toString()); + } + putNotNull(json, "msgType", msgType); + putNotNull(json, "arguments", arguments); + putNotNull(json, "result", result); + putNotNull(json, "error", error); + return eventInfo; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java new file mode 100644 index 0000000000..55ce036d9e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.StringUtils; + +@Data +@EqualsAndHashCode(callSuper = true) +@Schema +public class CalculatedFieldDebugEventFilter extends DebugEventFilter { + + @Schema(description = "String value representing the entity id in the event body", example = "57b6bafe-d600-423c-9267-fe31e5218986") + protected String entityId; + @Schema(description = "String value representing the entity type", allowableValues = "DEVICE") + protected String entityType; + @Schema(description = "String value representing the message id in the rule engine", example = "dcf44612-2ce4-4e5d-b462-ebb9c5628228") + protected String msgId; + @Schema(description = "String value representing the message type", example = "POST_TELEMETRY_REQUEST") + protected String msgType; + @Schema(description = "String value representing the arguments that were used in the calculation performed", + example = "{\"x\":{\"ts\":1739432016629,\"value\":20},\"y\":{\"ts\":1739429717656,\"value\":12}}") + protected String arguments; + @Schema(description = "String value representing the result of a calculation", + example = "{\"x + y\":54}") + protected String result; + + + @Override + public EventType getEventType() { + return EventType.DEBUG_CALCULATED_FIELD; + } + + @Override + public boolean isNotEmpty() { + return super.isNotEmpty() || !StringUtils.isEmpty(entityId) || !StringUtils.isEmpty(entityType) + || !StringUtils.isEmpty(msgId) || !StringUtils.isEmpty(msgType) + || !StringUtils.isEmpty(arguments) || !StringUtils.isEmpty(result); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java index 454d04f490..748771d1eb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java @@ -29,7 +29,8 @@ import io.swagger.v3.oas.annotations.media.Schema; @JsonSubTypes.Type(value = RuleChainDebugEventFilter.class, name = "DEBUG_RULE_CHAIN"), @JsonSubTypes.Type(value = ErrorEventFilter.class, name = "ERROR"), @JsonSubTypes.Type(value = LifeCycleEventFilter.class, name = "LC_EVENT"), - @JsonSubTypes.Type(value = StatisticsEventFilter.class, name = "STATS") + @JsonSubTypes.Type(value = StatisticsEventFilter.class, name = "STATS"), + @JsonSubTypes.Type(value = CalculatedFieldDebugEventFilter.class, name = "DEBUG_CALCULATED_FIELD") }) public interface EventFilter { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java index af75a92ea6..ce529c81bc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java @@ -22,7 +22,8 @@ public enum EventType { LC_EVENT("lc_event", "LC_EVENT"), STATS("stats_event", "STATS"), DEBUG_RULE_NODE("rule_node_debug_event", "DEBUG_RULE_NODE", true), - DEBUG_RULE_CHAIN("rule_chain_debug_event", "DEBUG_RULE_CHAIN", true); + DEBUG_RULE_CHAIN("rule_chain_debug_event", "DEBUG_RULE_CHAIN", true), + DEBUG_CALCULATED_FIELD("cf_debug_event", "DEBUG_CALCULATED_FIELD", true); @Getter private final String table; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java index caeef6bd2f..ed02d22a74 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java @@ -81,6 +81,10 @@ public class HousekeeperTask implements Serializable { return new TenantEntitiesDeletionHousekeeperTask(tenantId, entityType); } + public static HousekeeperTask deleteCalculatedFields(TenantId tenantId, EntityId entityId) { + return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_CALCULATED_FIELDS); + } + @JsonIgnore public String getDescription() { return taskType.getDescription() + " for " + entityId.getEntityType().getNormalName().toLowerCase() + " " + entityId.getId(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java index 1331b175ac..ef217debc3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java @@ -30,7 +30,8 @@ public enum HousekeeperTaskType { DELETE_ALARMS("alarms deletion"), UNASSIGN_ALARMS("alarms unassigning"), DELETE_TENANT_ENTITIES("tenant entities deletion"), - DELETE_ENTITIES("entities deletion"); + DELETE_ENTITIES("entities deletion"), + DELETE_CALCULATED_FIELDS("calculated fields deletion"); private final String description; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java new file mode 100644 index 0000000000..e17a066d88 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.id; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import org.thingsboard.server.common.data.EntityType; + +import java.io.Serial; +import java.util.UUID; + +@Schema +public class CalculatedFieldId extends UUIDBased implements EntityId { + + @Serial + private static final long serialVersionUID = 1L; + + @JsonCreator + public CalculatedFieldId(@JsonProperty("id") UUID id) { + super(id); + } + + public static CalculatedFieldId fromString(String calculatedFieldId) { + return new CalculatedFieldId(UUID.fromString(calculatedFieldId)); + } + + @Schema(requiredMode = Schema.RequiredMode.REQUIRED, description = "string", example = "CALCULATED_FIELD", allowableValues = "CALCULATED_FIELD") + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldLinkId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldLinkId.java new file mode 100644 index 0000000000..6a0c680bb6 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldLinkId.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.id; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import org.thingsboard.server.common.data.EntityType; + +import java.util.UUID; + +@Schema +public class CalculatedFieldLinkId extends UUIDBased implements EntityId { + + private static final long serialVersionUID = 1L; + + @JsonCreator + public CalculatedFieldLinkId(@JsonProperty("id") UUID id) { + super(id); + } + + public static CalculatedFieldLinkId fromString(String calculatedFieldLinkId) { + return new CalculatedFieldLinkId(UUID.fromString(calculatedFieldLinkId)); + } + + @Schema(requiredMode = Schema.RequiredMode.REQUIRED, description = "string", example = "CALCULATED_FIELD_LINK", allowableValues = "CALCULATED_FIELD_LINK") + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD_LINK; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index 7295f9795a..f5dd4b12a0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -113,6 +113,10 @@ public class EntityIdFactory { return new DomainId(uuid); case MOBILE_APP_BUNDLE: return new MobileAppBundleId(uuid); + case CALCULATED_FIELD: + return new CalculatedFieldId(uuid); + case CALCULATED_FIELD_LINK: + return new CalculatedFieldLinkId(uuid); } throw new IllegalArgumentException("EntityType " + type + " is not supported!"); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/UserAuthSettingsId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/UserAuthSettingsId.java index f17151dea9..a57ee7ee48 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/UserAuthSettingsId.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/UserAuthSettingsId.java @@ -15,11 +15,15 @@ */ package org.thingsboard.server.common.data.id; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + import java.util.UUID; public class UserAuthSettingsId extends UUIDBased { - public UserAuthSettingsId(UUID id) { + @JsonCreator + public UserAuthSettingsId(@JsonProperty("id") UUID id) { super(id); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TimeseriesSaveResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TimeseriesSaveResult.java new file mode 100644 index 0000000000..233eb8df73 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TimeseriesSaveResult.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.kv; + +import lombok.Data; + +import java.util.List; + +@Data(staticConstructor = "of") +public class TimeseriesSaveResult { + + public static final TimeseriesSaveResult EMPTY = new TimeseriesSaveResult(0, null); + + private final Integer dataPoints; + private final List versions; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java b/common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java index a25d12577f..db7f14171b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/limit/LimitedApi.java @@ -43,7 +43,8 @@ public enum LimitedApi { TRANSPORT_MESSAGES_PER_GATEWAY("transport messages per gateway", false), TRANSPORT_MESSAGES_PER_GATEWAY_DEVICE("transport messages per gateway device", false), EMAILS("emails sending", true), - WS_SUBSCRIPTIONS("WS subscriptions", false); + WS_SUBSCRIPTIONS("WS subscriptions", false), + CALCULATED_FIELD_DEBUG_EVENTS("calculated field debug events", true); private Function configExtractor; @Getter diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QuerySecurityContext.java b/common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java similarity index 72% rename from dao/src/main/java/org/thingsboard/server/dao/sql/query/QuerySecurityContext.java rename to common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java index bb5fdddb9d..ace36a1f37 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QuerySecurityContext.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sql.query; +package org.thingsboard.server.common.data.permission; import lombok.AllArgsConstructor; import lombok.Getter; @@ -21,8 +21,12 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + @AllArgsConstructor -public class QuerySecurityContext { +public class QueryContext { @Getter private final TenantId tenantId; @@ -33,7 +37,14 @@ public class QuerySecurityContext { @Getter private final boolean ignorePermissionCheck; - public QuerySecurityContext(TenantId tenantId, CustomerId customerId, EntityType entityType) { + @Getter + private final Map relatedParentIdMap = new HashMap<>(); + + public QueryContext(TenantId tenantId, CustomerId customerId, EntityType entityType) { this(tenantId, customerId, entityType, false); } + + public boolean isTenantUser() { + return customerId == null || customerId.isNullUid(); + } } \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/AlarmCountQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/AlarmCountQuery.java index 103bd90eac..71a8e8ef33 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/AlarmCountQuery.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/AlarmCountQuery.java @@ -17,7 +17,7 @@ package org.thingsboard.server.common.data.query; import lombok.AllArgsConstructor; import lombok.Builder; -import lombok.Getter; +import lombok.Data; import lombok.NoArgsConstructor; import lombok.ToString; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; @@ -29,7 +29,7 @@ import java.util.List; @Builder @NoArgsConstructor @AllArgsConstructor -@Getter +@Data @ToString public class AlarmCountQuery extends EntityCountQuery { private long startTs; @@ -40,4 +40,9 @@ public class AlarmCountQuery extends EntityCountQuery { private List severityList; private boolean searchPropagatedAlarms; private UserId assigneeId; + + public AlarmCountQuery(EntityFilter entityFilter) { + super(entityFilter); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java index 9ae56a1d60..8c4ae1e8ac 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.common.data.query; -import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import lombok.RequiredArgsConstructor; import org.thingsboard.server.common.data.validation.NoXss; @@ -26,7 +25,6 @@ import java.io.Serializable; @RequiredArgsConstructor public class DynamicValue implements Serializable { - @JsonIgnore private T resolvedValue; private final DynamicValueSourceType sourceType; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java index e2e7e18d9f..12b5331651 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.query; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Getter; import lombok.ToString; @@ -24,6 +25,7 @@ import java.util.List; @Schema @ToString +@JsonIgnoreProperties(ignoreUnknown = true) public class EntityCountQuery { @Getter diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java index 1519e3910d..328882bbd0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java @@ -16,20 +16,22 @@ package org.thingsboard.server.common.data.query; import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; import lombok.Data; -import lombok.RequiredArgsConstructor; +import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.id.EntityId; import java.util.Map; @Data -@RequiredArgsConstructor +@AllArgsConstructor +@NoArgsConstructor public class EntityData { - private final EntityId entityId; - private final Map> latest; - private final Map timeseries; - private final Map aggLatest; + private EntityId entityId; + private Map> latest; + private Map timeseries; + private Map aggLatest; public EntityData(EntityId entityId, Map> latest, Map timeseries) { this(entityId, latest, timeseries, null); @@ -44,4 +46,5 @@ public class EntityData { aggLatest.clear(); } } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java index 4be9633d96..5507f53f08 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityFilter.java @@ -39,7 +39,8 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(value = AssetSearchQueryFilter.class, name = "assetSearchQuery"), @JsonSubTypes.Type(value = DeviceSearchQueryFilter.class, name = "deviceSearchQuery"), @JsonSubTypes.Type(value = EntityViewSearchQueryFilter.class, name = "entityViewSearchQuery"), - @JsonSubTypes.Type(value = EdgeSearchQueryFilter.class, name = "edgeSearchQuery")}) + @JsonSubTypes.Type(value = EdgeSearchQueryFilter.class, name = "edgeSearchQuery") +}) public interface EntityFilter { @JsonIgnore diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/queue/ProcessingStrategyType.java b/common/data/src/main/java/org/thingsboard/server/common/data/queue/ProcessingStrategyType.java index 5b8c86d7d1..ca63d34a84 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/queue/ProcessingStrategyType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/queue/ProcessingStrategyType.java @@ -13,21 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.thingsboard.server.common.data.queue; public enum ProcessingStrategyType { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueConfig.java index 9683967659..294a7ee2c2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueConfig.java @@ -15,10 +15,22 @@ */ package org.thingsboard.server.common.data.queue; +import lombok.Data; + public interface QueueConfig { boolean isConsumerPerPartition(); int getPollInterval(); + static QueueConfig of(boolean consumerPerPartition, long pollInterval) { + return new BasicQueueConfig(consumerPerPartition, (int) pollInterval); + } + + @Data + class BasicQueueConfig implements QueueConfig { + private final boolean consumerPerPartition; + private final int pollInterval; + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java b/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java index 7cbedcbacb..8980d0e634 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java @@ -25,6 +25,8 @@ import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.BaseDataWithAdditionalInfo; import org.thingsboard.server.common.data.HasVersion; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsObject; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.validation.Length; @@ -34,7 +36,7 @@ import java.io.Serializable; @Schema @EqualsAndHashCode(exclude = "additionalInfoBytes") @ToString(exclude = {"additionalInfoBytes"}) -public class EntityRelation implements HasVersion, Serializable { +public class EntityRelation implements HasVersion, Serializable, EdqsObject { private static final long serialVersionUID = 2807343040519543363L; @@ -107,7 +109,7 @@ public class EntityRelation implements HasVersion, Serializable { return typeGroup; } - @Schema(description = "Additional parameters of the relation",implementation = com.fasterxml.jackson.databind.JsonNode.class) + @Schema(description = "Additional parameters of the relation", implementation = com.fasterxml.jackson.databind.JsonNode.class) public JsonNode getAdditionalInfo() { return BaseDataWithAdditionalInfo.getJson(() -> additionalInfo, () -> additionalInfoBytes); } @@ -116,4 +118,19 @@ public class EntityRelation implements HasVersion, Serializable { BaseDataWithAdditionalInfo.setJson(addInfo, json -> this.additionalInfo = json, bytes -> this.additionalInfoBytes = bytes); } + @JsonIgnore + public String key() { + return "r_" + from + "_" + to + "_" + typeGroup + "_" + type; + } + + @Override + public Long version() { + return version; + } + + @Override + public ObjectType type() { + return ObjectType.RELATION; + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java index bfa82caa2b..98948ca642 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java @@ -38,4 +38,5 @@ public class DeviceExportData extends EntityExportData { public boolean hasCredentials() { return credentials != null; } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index 7bc9a63c29..072be6acf2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -26,6 +26,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo.As; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.sync.JsonTbEntity; @@ -55,6 +56,8 @@ public class EntityExportData> { public static final Comparator attrComparator = Comparator .comparing(AttributeExportData::getKey).thenComparing(AttributeExportData::getLastUpdateTs); + public static final Comparator calculatedFieldsComparator = Comparator.comparing(CalculatedField::getName); + @JsonProperty(index = 2) @JsonTbEntity private E entity; @@ -65,6 +68,9 @@ public class EntityExportData> { private List relations; @JsonProperty(index = 101) private Map> attributes; + @JsonProperty(index = 102) + @JsonIgnoreProperties({"id", "entityId", "createdTime", "version"}) + private List calculatedFields; public EntityExportData sort() { if (relations != null && !relations.isEmpty()) { @@ -73,6 +79,9 @@ public class EntityExportData> { if (attributes != null && !attributes.isEmpty()) { attributes.values().forEach(list -> list.sort(attrComparator)); } + if (calculatedFields != null && !calculatedFields.isEmpty()) { + calculatedFields.sort(calculatedFieldsComparator); + } return this; } @@ -96,4 +105,9 @@ public class EntityExportData> { return relations != null; } + @JsonIgnore + public boolean hasCalculatedFields() { + return calculatedFields != null; + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java index 46dc6860ee..078ca29e28 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java @@ -25,7 +25,10 @@ import lombok.NoArgsConstructor; @NoArgsConstructor @Builder public class EntityExportSettings { + private boolean exportRelations; private boolean exportAttributes; private boolean exportCredentials; + private boolean exportCalculatedFields; + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java index 2abdb55b65..d49801537c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java @@ -25,8 +25,11 @@ import lombok.NoArgsConstructor; @NoArgsConstructor @Builder public class EntityImportSettings { + private boolean findExistingByName; private boolean updateRelations; private boolean saveAttributes; private boolean saveCredentials; + private boolean saveCalculatedFields; + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java index 5a02b18364..6d1ac2283b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java @@ -17,13 +17,18 @@ package org.thingsboard.server.common.data.sync.vc.request.create; import lombok.Data; +import java.io.Serial; import java.io.Serializable; @Data public class VersionCreateConfig implements Serializable { + + @Serial private static final long serialVersionUID = 1223723167716612772L; private boolean saveRelations; private boolean saveAttributes; private boolean saveCredentials; + private boolean saveCalculatedFields; + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java index 0ae263dada..7f3ce89372 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java @@ -23,5 +23,6 @@ public class VersionLoadConfig { private boolean loadRelations; private boolean loadAttributes; private boolean loadCredentials; + private boolean loadCalculatedFields; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index f62b3aaa9f..f256b02d9a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.tenant.profile; import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.Min; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -135,6 +136,19 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private double warnThreshold; + @Schema(example = "5") + private long maxCalculatedFieldsPerEntity = 5; + @Schema(example = "10") + private long maxArgumentsPerCF = 10; + @Builder.Default + @Min(value = 1, message = "must be at least 1") + @Schema(example = "1000") + private long maxDataPointsPerRollingArg = 1000; + @Schema(example = "32") + private long maxStateSizeInKBytes = 32; + @Schema(example = "2") + private long maxSingleValueArgumentSizeInKBytes = 2; + @Override public long getProfileThreshold(ApiUsageRecordKey key) { return switch (key) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java index b1f6c27fd8..44ca79cabb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.tenant.profile; import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.Valid; import lombok.Data; import java.io.Serializable; @@ -27,6 +28,7 @@ public class TenantProfileData implements Serializable { private static final long serialVersionUID = -3642550257035920976L; + @Valid @Schema(description = "Complex JSON object that contains profile settings: max devices, max assets, rate limits, etc.") private TenantProfileConfiguration configuration; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java index 390c1f234f..71c5256203 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java @@ -17,6 +17,7 @@ package org.thingsboard.server.common.data.util; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -75,4 +76,23 @@ public class CollectionsUtil { return isEmpty(collection) || collection.contains(element); } + public static HashSet concat(Set set1, Set set2) { + HashSet result = new HashSet<>(); + result.addAll(set1); + result.addAll(set2); + return result; + } + + public static boolean isOneOf(V value, V... others) { + if (value == null) { + return false; + } + for (V other : others) { + if (value.equals(other)) { + return true; + } + } + return false; + } + } diff --git a/common/edge-api/src/main/proto/edge.proto b/common/edge-api/src/main/proto/edge.proto index 6a7482a3c1..023ac00634 100644 --- a/common/edge-api/src/main/proto/edge.proto +++ b/common/edge-api/src/main/proto/edge.proto @@ -42,6 +42,8 @@ enum EdgeVersion { V_3_8_0 = 8; V_3_9_0 = 9; V_4_0_0 = 10; + + V_LATEST = 999; } /** @@ -303,7 +305,9 @@ message NotificationTemplateUpdateMsg { optional string entity = 4; } +// DEPRECATED. FOR REMOVAL message RuleChainMetadataRequestMsg { + option deprecated = true; int64 ruleChainIdMSB = 1; int64 ruleChainIdLSB = 2; } @@ -321,22 +325,30 @@ message RelationRequestMsg { string entityType = 3; } +// DEPRECATED. FOR REMOVAL message UserCredentialsRequestMsg { + option deprecated = true; int64 userIdMSB = 1; int64 userIdLSB = 2; } +// DEPRECATED. FOR REMOVAL message DeviceCredentialsRequestMsg { + option deprecated = true; int64 deviceIdMSB = 1; int64 deviceIdLSB = 2; } +// DEPRECATED. FOR REMOVAL message WidgetBundleTypesRequestMsg { + option deprecated = true; int64 widgetBundleIdMSB = 1; int64 widgetBundleIdLSB = 2; } +// DEPRECATED. FOR REMOVAL message EntityViewsRequestMsg { + option deprecated = true; int64 entityIdMSB = 1; int64 entityIdLSB = 2; string entityType = 3; @@ -394,14 +406,14 @@ message UplinkMsg { repeated DeviceCredentialsUpdateMsg deviceCredentialsUpdateMsg = 4; repeated AlarmUpdateMsg alarmUpdateMsg = 5; repeated RelationUpdateMsg relationUpdateMsg = 6; - repeated RuleChainMetadataRequestMsg ruleChainMetadataRequestMsg = 7; + repeated RuleChainMetadataRequestMsg ruleChainMetadataRequestMsg = 7 [deprecated = true]; repeated AttributesRequestMsg attributesRequestMsg = 8; repeated RelationRequestMsg relationRequestMsg = 9; - repeated UserCredentialsRequestMsg userCredentialsRequestMsg = 10; - repeated DeviceCredentialsRequestMsg deviceCredentialsRequestMsg = 11; + repeated UserCredentialsRequestMsg userCredentialsRequestMsg = 10 [deprecated = true]; + repeated DeviceCredentialsRequestMsg deviceCredentialsRequestMsg = 11 [deprecated = true]; repeated DeviceRpcCallMsg deviceRpcCallMsg = 12; - repeated WidgetBundleTypesRequestMsg widgetBundleTypesRequestMsg = 14; - repeated EntityViewsRequestMsg entityViewsRequestMsg = 15; + repeated WidgetBundleTypesRequestMsg widgetBundleTypesRequestMsg = 14 [deprecated = true]; + repeated EntityViewsRequestMsg entityViewsRequestMsg = 15 [deprecated = true]; repeated AssetUpdateMsg assetUpdateMsg = 16; repeated DashboardUpdateMsg dashboardUpdateMsg = 17; repeated EntityViewUpdateMsg entityViewUpdateMsg = 18; @@ -409,6 +421,8 @@ message UplinkMsg { repeated DeviceProfileUpdateMsg deviceProfileUpdateMsg = 20; repeated ResourceUpdateMsg resourceUpdateMsg = 21; repeated AlarmCommentUpdateMsg alarmCommentUpdateMsg = 22; + repeated RuleChainUpdateMsg ruleChainUpdateMsg = 23; + repeated RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = 24; } message UplinkResponseMsg { @@ -427,7 +441,7 @@ message DownlinkMsg { int32 downlinkMsgId = 1; SyncCompletedMsg syncCompletedMsg = 2; repeated EntityDataProto entityData = 3; - repeated DeviceCredentialsRequestMsg deviceCredentialsRequestMsg = 4; + repeated DeviceCredentialsRequestMsg deviceCredentialsRequestMsg = 4 [deprecated = true]; repeated DeviceUpdateMsg deviceUpdateMsg = 5; repeated DeviceProfileUpdateMsg deviceProfileUpdateMsg = 6; repeated DeviceCredentialsUpdateMsg deviceCredentialsUpdateMsg = 7; diff --git a/common/edqs/pom.xml b/common/edqs/pom.xml new file mode 100644 index 0000000000..f7106e56fd --- /dev/null +++ b/common/edqs/pom.xml @@ -0,0 +1,97 @@ + + + 4.0.0 + + org.thingsboard + 4.0.0-SNAPSHOT + common + + org.thingsboard.common + edqs + jar + + ThingsBoard EDQS API + https://thingsboard.io + + + UTF-8 + ${basedir}/../.. + + + + + org.rocksdb + rocksdbjni + + + org.thingsboard.common + proto + + + org.thingsboard.common + data + + + org.thingsboard.common + util + + + org.thingsboard.common + message + + + org.thingsboard.common + stats + + + org.thingsboard.common + cluster-api + + + org.thingsboard.common + queue + + + org.springframework.boot + spring-boot-starter-web + + + com.github.ben-manes.caffeine + caffeine + + + org.springframework + spring-context-support + + + org.springframework.boot + spring-boot-autoconfigure + + + + + + thingsboard-repo-deploy + ThingsBoard Repo Deployment + https://repo.thingsboard.io/artifactory/libs-release-public + + + + diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ApiUsageStateData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ApiUsageStateData.java new file mode 100644 index 0000000000..f7cd51fc38 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ApiUsageStateData.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.ApiUsageStateFields; + +import java.util.UUID; + +@ToString(callSuper = true) +public class ApiUsageStateData extends BaseEntityData { + + public ApiUsageStateData(UUID entityId) { + super(entityId); + } + + @Override + public EntityType getEntityType() { + return EntityType.API_USAGE_STATE; + } + + @Override + public String getEntityName() { + return getEntityOwnerName(); + } + + @Override + public String getEntityOwnerName() { + return repo.getOwnerName(fields.getEntityId()); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/AssetData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/AssetData.java new file mode 100644 index 0000000000..52e6151ff3 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/AssetData.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.AssetFields; + +import java.util.UUID; + +@ToString(callSuper = true) +public class AssetData extends ProfileAwareData { + + public AssetData(UUID id) { + super(id); + } + + @Override + public EntityType getEntityType() { + return EntityType.ASSET; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java new file mode 100644 index 0000000000..10ee17fc75 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java @@ -0,0 +1,180 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.edqs.data.dp.BoolDataPoint; +import org.thingsboard.server.common.data.edqs.DataPoint; +import org.thingsboard.server.edqs.data.dp.LongDataPoint; +import org.thingsboard.server.edqs.data.dp.StringDataPoint; +import org.thingsboard.server.edqs.query.DataKey; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +@ToString +public abstract class BaseEntityData implements EntityData { + + @Getter + private final UUID id; + @Getter + protected final Map serverAttrMap; + @Getter + private final Map tMap; + + @Getter + @Setter + private volatile UUID customerId; + + @Setter + protected TenantRepo repo; + + @Getter + @Setter + protected volatile T fields; + + public BaseEntityData(UUID id) { + this.id = id; + this.serverAttrMap = new ConcurrentHashMap<>(); + this.tMap = new ConcurrentHashMap<>(); + } + + @Override + public DataPoint getAttr(Integer keyId, EntityKeyType entityKeyType) { + return switch (entityKeyType) { + case ATTRIBUTE, SERVER_ATTRIBUTE -> serverAttrMap.get(keyId); + default -> null; + }; + } + + @Override + public boolean putAttr(Integer keyId, AttributeScope scope, DataPoint value) { + return serverAttrMap.put(keyId, value) == null; + } + + @Override + public boolean removeAttr(Integer keyId, AttributeScope scope) { + return serverAttrMap.remove(keyId) != null; + } + + @Override + public DataPoint getTs(Integer keyId) { + return tMap.get(keyId); + } + + @Override + public boolean putTs(Integer keyId, DataPoint value) { + return tMap.put(keyId, value) == null; + } + + @Override + public boolean removeTs(Integer keyId) { + return tMap.remove(keyId) != null; + } + + @Override + public EntityType getOwnerType() { + return customerId != null ? EntityType.CUSTOMER : EntityType.TENANT; + } + + @Override + public DataPoint getDataPoint(DataKey key, QueryContext ctx) { + return switch (key.type()) { + case TIME_SERIES -> getTs(key.keyId()); + case ATTRIBUTE, SERVER_ATTRIBUTE, CLIENT_ATTRIBUTE, SHARED_ATTRIBUTE -> getAttr(key.keyId(), key.type()); + case ENTITY_FIELD -> getField(key, ctx); + default -> throw new RuntimeException(key.type() + " not supported"); + }; + } + + private DataPoint getField(DataKey newKey, QueryContext ctx) { + if (fields == null) { + return null; + } + String key = newKey.key(); + return switch (key) { + case "createdTime" -> new LongDataPoint(System.currentTimeMillis(), fields.getCreatedTime()); + case "edgeTemplate" -> new BoolDataPoint(System.currentTimeMillis(), fields.isEdgeTemplate()); + case "parentId" -> new StringDataPoint(System.currentTimeMillis(), getRelatedParentId(ctx)); + default -> new StringDataPoint(System.currentTimeMillis(), getField(key), false); + }; + } + + @Override + public String getField(String name) { + if (fields == null) { + return null; + } + return switch (name) { + case "name" -> getEntityName(); + case "ownerName" -> getEntityOwnerName(); + case "ownerType" -> customerId != null ? EntityType.CUSTOMER.name() : EntityType.TENANT.name(); + case "entityType" -> Optional.ofNullable(getEntityType()).map(EntityType::name).orElse(""); + default -> fields.getAsString(name); + }; + } + + public String getEntityOwnerName() { + return repo.getOwnerName(getCustomerId() == null || CustomerId.NULL_UUID.equals(getCustomerId()) ? null : + new CustomerId(getCustomerId())); + } + + public String getEntityName() { + return getFields().getName(); + } + + private String getRelatedParentId(QueryContext ctx) { + return Optional.ofNullable(ctx.getRelatedParentIdMap().get(getId())) + .map(UUID::toString) + .orElse(""); + } + + @Override + public EntityType getEntityType() { + return null; + } + + @Override + public boolean isEmpty() { + return fields == null; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BaseEntityData that = (BaseEntityData) o; + return Objects.equals(id, that.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java new file mode 100644 index 0000000000..bf2a3f6da7 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.CustomerFields; + +import java.util.Collection; +import java.util.Collections; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class CustomerData extends BaseEntityData { + + private final ConcurrentMap>> entitiesById = new ConcurrentHashMap<>(); + + public CustomerData(UUID entityId) { + super(entityId); + } + + @Override + public EntityType getEntityType() { + return EntityType.CUSTOMER; + } + + public Collection> getEntities(EntityType entityType) { + var map = entitiesById.get(entityType); + if (map == null) { + return Collections.emptyList(); + } else { + return map.values(); + } + } + + public void addOrUpdate(EntityData ed) { + entitiesById.computeIfAbsent(ed.getEntityType(), et -> new ConcurrentHashMap<>()).put(ed.getId(), ed); + } + + public boolean remove(EntityData ed) { + var map = entitiesById.get(ed.getEntityType()); + if (map != null) { + return map.remove(ed.getId()) != null; + } else { + return false; + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/DeviceData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/DeviceData.java new file mode 100644 index 0000000000..3a3e5c5792 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/DeviceData.java @@ -0,0 +1,86 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.ToString; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.DeviceFields; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.edqs.DataPoint; + +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +@ToString(callSuper = true) +public class DeviceData extends ProfileAwareData { + + private final Map clientAttrMap; + private final Map sharedAttrMap; + + public DeviceData(UUID entityId) { + super(entityId); + this.clientAttrMap = new ConcurrentHashMap<>(); + this.sharedAttrMap = new ConcurrentHashMap<>(); + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE; + } + + @Override + public DataPoint getAttr(Integer keyId, EntityKeyType entityKeyType) { + return switch (entityKeyType) { + case ATTRIBUTE -> getAttributeDataPoint(keyId); + case SERVER_ATTRIBUTE -> serverAttrMap.get(keyId); + case CLIENT_ATTRIBUTE -> clientAttrMap.get(keyId); + case SHARED_ATTRIBUTE -> sharedAttrMap.get(keyId); + default -> throw new RuntimeException(entityKeyType + " not implemented"); + }; + } + + @Override + public boolean putAttr(Integer keyId, AttributeScope scope, DataPoint value) { + return switch (scope) { + case SERVER_SCOPE -> serverAttrMap.put(keyId, value) == null; + case CLIENT_SCOPE -> clientAttrMap.put(keyId, value) == null; + case SHARED_SCOPE -> sharedAttrMap.put(keyId, value) == null; + }; + } + + @Override + public boolean removeAttr(Integer keyId, AttributeScope scope) { + return switch (scope) { + case SERVER_SCOPE -> serverAttrMap.remove(keyId) != null; + case CLIENT_SCOPE -> clientAttrMap.remove(keyId) != null; + case SHARED_SCOPE -> sharedAttrMap.remove(keyId) != null; + }; + } + + private DataPoint getAttributeDataPoint(Integer keyId) { + DataPoint dp = serverAttrMap.get(keyId); + if (dp == null) { + dp = sharedAttrMap.get(keyId); + if (dp == null) { + dp = clientAttrMap.get(keyId); + } + } + return dp; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityData.java new file mode 100644 index 0000000000..53ee73f638 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityData.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.edqs.DataPoint; +import org.thingsboard.server.edqs.query.DataKey; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.UUID; + +public interface EntityData { + + UUID getId(); + + EntityType getEntityType(); + + UUID getCustomerId(); + + void setCustomerId(UUID customerId); + + void setRepo(TenantRepo repo); + + T getFields(); + + void setFields(T fields); + + DataPoint getAttr(Integer keyId, EntityKeyType entityKeyType); + + boolean putAttr(Integer keyId, AttributeScope scope, DataPoint value); + + boolean removeAttr(Integer keyId, AttributeScope scope); + + DataPoint getTs(Integer keyId); + + boolean putTs(Integer keyId, DataPoint value); + + boolean removeTs(Integer keyId); + + EntityType getOwnerType(); + + DataPoint getDataPoint(DataKey key, QueryContext queryContext); + + String getField(String name); + + boolean isEmpty(); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityProfileData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityProfileData.java new file mode 100644 index 0000000000..a13c70557b --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityProfileData.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; + +import java.util.UUID; + +@ToString(callSuper = true) +public class EntityProfileData extends BaseEntityData { + + private final EntityType entityType; + + public EntityProfileData(UUID entityId, EntityType entityType) { + super(entityId); + this.entityType = entityType; + } + + @Override + public EntityType getEntityType() { + return entityType; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/GenericData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/GenericData.java new file mode 100644 index 0000000000..344a2b7049 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/GenericData.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; + +import java.util.UUID; + +@ToString(callSuper = true) +public class GenericData extends BaseEntityData { + + private final EntityType entityType; + + public GenericData(EntityType entityType, UUID entityId) { + super(entityId); + this.entityType = entityType; + } + + @Override + public EntityType getEntityType() { + return entityType; + } +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ProfileAwareData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ProfileAwareData.java new file mode 100644 index 0000000000..bab0f962c4 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ProfileAwareData.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import org.thingsboard.server.common.data.edqs.fields.ProfileAwareFields; + +import java.util.UUID; + +public abstract class ProfileAwareData extends BaseEntityData { + + public ProfileAwareData(UUID id) { + super(id); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationData.java new file mode 100644 index 0000000000..e006def91b --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationData.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; + +import java.util.UUID; + +public record RelationData(UUID fromId, EntityType fromType, UUID toId, EntityType toType, String type, + RelationTypeGroup typeGroup) { + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationInfo.java similarity index 77% rename from application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java rename to common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationInfo.java index 4911bb9296..5eb5d7aee4 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationInfo.java @@ -13,12 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue.ruleengine; +package org.thingsboard.server.edqs.data; -import java.io.Serializable; +import lombok.Data; -public enum QueueEvent implements Serializable { +@Data +public class RelationInfo { - PARTITION_CHANGE, CONFIG_UPDATE, DELETE + private final String type; + private final EntityData target; } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationsRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationsRepo.java new file mode 100644 index 0000000000..c094e37261 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationsRepo.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.NoArgsConstructor; + +import java.util.Collections; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +@NoArgsConstructor +public class RelationsRepo { + + private final ConcurrentMap> fromRelations = new ConcurrentHashMap<>(); + private final ConcurrentMap> toRelations = new ConcurrentHashMap<>(); + + public boolean add(EntityData from, EntityData to, String type) { + boolean addedFromRelation = fromRelations.computeIfAbsent(from.getId(), k -> ConcurrentHashMap.newKeySet()).add(new RelationInfo(type, to)); + boolean addedToRelation = toRelations.computeIfAbsent(to.getId(), k -> ConcurrentHashMap.newKeySet()).add(new RelationInfo(type, from)); + return addedFromRelation || addedToRelation; + } + + public Set getFrom(UUID entityId) { + var result = fromRelations.get(entityId); + return result == null ? Collections.emptySet() : result; + } + + public Set getTo(UUID entityId) { + var result = toRelations.get(entityId); + return result == null ? Collections.emptySet() : result; + } + + public boolean remove(UUID from, UUID to, String type) { + boolean removedFromRelation = false; + boolean removedToRelation = false; + Set fromRelations = this.fromRelations.get(from); + if (fromRelations != null) { + removedFromRelation = fromRelations.removeIf(relationInfo -> relationInfo.getTarget().getId().equals(to) && relationInfo.getType().equals(type)); + } + Set toRelations = this.toRelations.get(to); + if (toRelations != null) { + removedToRelation = toRelations.removeIf(relationInfo -> relationInfo.getTarget().getId().equals(from) && relationInfo.getType().equals(type)); + } + return removedFromRelation || removedToRelation; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/TenantData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/TenantData.java new file mode 100644 index 0000000000..6822856edf --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/TenantData.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.TenantFields; + +import java.util.UUID; + +public class TenantData extends BaseEntityData { + + public TenantData(UUID entityId) { + super(entityId); + } + + @Override + public EntityType getEntityType() { + return EntityType.TENANT; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java new file mode 100644 index 0000000000..fd2d099281 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.edqs.DataPoint; + +@RequiredArgsConstructor +public abstract class AbstractDataPoint implements DataPoint { + + @Getter + private final long ts; + + @Override + public String getStr() { + throw new RuntimeException(NOT_SUPPORTED); + } + + @Override + public long getLong() { + throw new RuntimeException(NOT_SUPPORTED); + } + + @Override + public double getDouble() { + throw new RuntimeException(NOT_SUPPORTED); + } + + @Override + public boolean getBool() { + throw new RuntimeException(NOT_SUPPORTED); + } + + @Override + public String getJson() { + throw new RuntimeException(NOT_SUPPORTED); + } + + public String toString() { + return valueToString(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java new file mode 100644 index 0000000000..83d91d8f75 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import org.thingsboard.server.common.data.kv.DataType; + +public class BoolDataPoint extends AbstractDataPoint { + + @Getter + private final boolean value; + + public BoolDataPoint(long ts, boolean value) { + super(ts); + this.value = value; + } + + @Override + public DataType getType() { + return DataType.BOOLEAN; + } + + @Override + public boolean getBool() { + return value; + } + + @Override + public String valueToString() { + return Boolean.toString(value); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java new file mode 100644 index 0000000000..bce9d86875 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import org.thingsboard.server.common.data.kv.DataType; + +public class CompressedJsonDataPoint extends CompressedStringDataPoint { + + public CompressedJsonDataPoint(long ts, byte[] compressedValue) { + super(ts, compressedValue); + } + + @Override + public DataType getType() { + return DataType.JSON; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java new file mode 100644 index 0000000000..634b63e012 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import lombok.SneakyThrows; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.edqs.util.TbBytePool; +import org.xerial.snappy.Snappy; + +public class CompressedStringDataPoint extends AbstractDataPoint { + + public static final int MIN_STR_SIZE_TO_COMPRESS = 512; + @Getter + private final byte[] compressedValue; + + @SneakyThrows + public CompressedStringDataPoint(long ts, byte[] compressedValue) { + super(ts); + this.compressedValue = TbBytePool.intern(compressedValue); + } + + @Override + public DataType getType() { + return DataType.STRING; + } + + @SneakyThrows + @Override + public String getStr() { + return Snappy.uncompressString(compressedValue); + } + + @Override + public String valueToString() { + return getStr(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java new file mode 100644 index 0000000000..21b355bc46 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import org.thingsboard.server.common.data.kv.DataType; + +public class DoubleDataPoint extends AbstractDataPoint { + + @Getter + private final double value; + + public DoubleDataPoint(long ts, double value) { + super(ts); + this.value = value; + } + + @Override + public DataType getType() { + return DataType.DOUBLE; + } + + @Override + public double getDouble() { + return value; + } + + @Override + public String valueToString() { + return Double.toString(value); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java new file mode 100644 index 0000000000..3a8d570f43 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.edqs.util.TbStringPool; + +public class JsonDataPoint extends AbstractDataPoint { + + @Getter + private final String value; + + public JsonDataPoint(long ts, String value) { + super(ts); + this.value = TbStringPool.intern(value); + } + + @Override + public DataType getType() { + return DataType.JSON; + } + + @Override + public String getJson() { + return value; + } + + @Override + public String valueToString() { + return value; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java new file mode 100644 index 0000000000..7fbe90e814 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import org.thingsboard.server.common.data.kv.DataType; + +public class LongDataPoint extends AbstractDataPoint { + + @Getter + private final long value; + + public LongDataPoint(long ts, long value) { + super(ts); + this.value = value; + } + + @Override + public DataType getType() { + return DataType.LONG; + } + + @Override + public long getLong() { + return value; + } + + @Override + public double getDouble() { + return value; + } + + @Override + public String valueToString() { + return Long.toString(value); + } +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java new file mode 100644 index 0000000000..54156500fe --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.edqs.util.TbStringPool; + +public class StringDataPoint extends AbstractDataPoint { + + @Getter + private final String value; + + public StringDataPoint(long ts, String value) { + this(ts, value, true); + } + + public StringDataPoint(long ts, String value, boolean deduplicate) { + super(ts); + this.value = deduplicate ? TbStringPool.intern(value) : value; + } + + @Override + public DataType getType() { + return DataType.STRING; + } + + @Override + public String getStr() { + return value; + } + + @Override + public String valueToString() { + return value; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java new file mode 100644 index 0000000000..c5696c08f7 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java @@ -0,0 +1,298 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.processor; + +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ExceptionUtil; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEvent; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.query.EdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsResponse; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.util.CollectionsUtil; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.edqs.repo.EdqsRepository; +import org.thingsboard.server.edqs.state.EdqsPartitionService; +import org.thingsboard.server.edqs.state.EdqsStateService; +import org.thingsboard.server.edqs.util.EdqsConverter; +import org.thingsboard.server.edqs.util.VersionsStore; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueHandler; +import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.edqs.EdqsComponent; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsConfig.EdqsPartitioningStrategy; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.edqs.EdqsQueueFactory; +import org.thingsboard.server.queue.util.AfterStartUp; + +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTopic; + +@EdqsComponent +@Service +@RequiredArgsConstructor +@Slf4j +public class EdqsProcessor implements TbQueueHandler, TbProtoQueueMsg> { + + private final EdqsQueueFactory queueFactory; + private final EdqsConverter converter; + private final EdqsRepository repository; + private final EdqsConfig config; + private final EdqsPartitionService partitionService; + private final ConfigurableApplicationContext applicationContext; + private final EdqsStateService stateService; + + private PartitionedQueueConsumerManager> eventConsumer; + private TbQueueResponseTemplate, TbProtoQueueMsg> responseTemplate; + + private ExecutorService consumersExecutor; + private ExecutorService taskExecutor; + private ScheduledExecutorService scheduler; + private ListeningExecutorService requestExecutor; + + private final VersionsStore versionsStore = new VersionsStore(); + + private final AtomicInteger counter = new AtomicInteger(); + + @Getter + private Consumer errorHandler; + + @PostConstruct + private void init() { + consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("edqs-consumer")); + taskExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "edqs-consumer-task-executor"); + scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-scheduler"); + requestExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(12, "edqs-requests")); + errorHandler = error -> { + if (error instanceof OutOfMemoryError) { + log.error("OOM detected, shutting down"); + repository.clear(); + Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("edqs-shutdown")) + .execute(applicationContext::close); + } + }; + + eventConsumer = PartitionedQueueConsumerManager.>create() + .queueKey(new QueueKey(ServiceType.EDQS, EdqsQueue.EVENTS.getTopic())) + .topic(EdqsQueue.EVENTS.getTopic()) + .pollInterval(config.getPollInterval()) + .msgPackProcessor((msgs, consumer, config) -> { + for (TbProtoQueueMsg queueMsg : msgs) { + if (consumer.isStopped()) { + return; + } + try { + ToEdqsMsg msg = queueMsg.getValue(); + process(msg, EdqsQueue.EVENTS); + } catch (Exception t) { + log.error("Failed to process message: {}", queueMsg, t); + } + } + consumer.commit(); + }) + .consumerCreator((config, partitionId) -> queueFactory.createEdqsMsgConsumer(EdqsQueue.EVENTS)) + .consumerExecutor(consumersExecutor) + .taskExecutor(taskExecutor) + .scheduler(scheduler) + .uncaughtErrorHandler(errorHandler) + .build(); + stateService.init(eventConsumer); + + responseTemplate = queueFactory.createEdqsResponseTemplate(); + } + + @AfterStartUp(order = 1) + public void start() { + responseTemplate.launch(this); + } + + @EventListener + public void onPartitionsChange(PartitionChangeEvent event) { + if (event.getServiceType() != ServiceType.EDQS) { + return; + } + try { + Set newPartitions = event.getNewPartitions().get(new QueueKey(ServiceType.EDQS)); + Set partitions = newPartitions.stream() + .map(tpi -> tpi.withUseInternalPartition(true)) + .collect(Collectors.toSet()); + + stateService.process(withTopic(partitions, EdqsQueue.STATE.getTopic())); + // eventsConsumer's partitions are updated by stateService + responseTemplate.subscribe(withTopic(partitions, config.getRequestsTopic())); // FIXME: we subscribe to partitions before we are ready. implement consumer-per-partition version for request template + + Set oldPartitions = event.getOldPartitions().get(new QueueKey(ServiceType.EDQS)); + if (CollectionsUtil.isNotEmpty(oldPartitions)) { + Set removedPartitions = Sets.difference(oldPartitions, newPartitions).stream() + .map(tpi -> tpi.getPartition().orElse(-1)).collect(Collectors.toSet()); + if (config.getPartitioningStrategy() != EdqsPartitioningStrategy.TENANT && !removedPartitions.isEmpty()) { + log.warn("Partitions {} were removed but shouldn't be (due to NONE partitioning strategy)", removedPartitions); + } + repository.clearIf(tenantId -> { + Integer partition = partitionService.resolvePartition(tenantId); + return partition != null && removedPartitions.contains(partition); + }); + } + } catch (Throwable t) { + log.error("Failed to handle partition change event {}", event, t); + } + } + + @Override + public ListenableFuture> handle(TbProtoQueueMsg queueMsg) { + ToEdqsMsg toEdqsMsg = queueMsg.getValue(); + return requestExecutor.submit(() -> { + EdqsRequest request; + TenantId tenantId; + CustomerId customerId; + try { + request = Objects.requireNonNull(JacksonUtil.fromString(toEdqsMsg.getRequestMsg().getValue(), EdqsRequest.class)); + tenantId = getTenantId(toEdqsMsg); + customerId = getCustomerId(toEdqsMsg); + } catch (Exception e) { + log.error("Failed to parse request msg: {}", toEdqsMsg, e); + throw e; + } + + EdqsResponse response = processRequest(tenantId, customerId, request); + return new TbProtoQueueMsg<>(queueMsg.getKey(), FromEdqsMsg.newBuilder() + .setResponseMsg(TransportProtos.EdqsResponseMsg.newBuilder() + .setValue(JacksonUtil.toString(response)) + .build()) + .build(), queueMsg.getHeaders()); + }); + } + + private EdqsResponse processRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) { + EdqsResponse response = new EdqsResponse(); + try { + if (request.getEntityDataQuery() != null) { + PageData result = repository.findEntityDataByQuery(tenantId, customerId, + request.getEntityDataQuery(), false); + response.setEntityDataQueryResult(result.mapData(QueryResult::toOldEntityData)); + } else if (request.getEntityCountQuery() != null) { + long result = repository.countEntitiesByQuery(tenantId, customerId, request.getEntityCountQuery(), tenantId.isSysTenantId()); + response.setEntityCountQueryResult(result); + } + log.trace("[{}] Request: {}, response: {}", tenantId, request, response); + } catch (Throwable e) { + log.error("[{}] Failed to process request: {}", tenantId, request, e); + response.setError(ExceptionUtil.getMessage(e)); + } + return response; + } + + public void process(ToEdqsMsg edqsMsg, EdqsQueue queue) { + log.trace("Processing message: {}", edqsMsg); + if (edqsMsg.hasEventMsg()) { + EdqsEventMsg eventMsg = edqsMsg.getEventMsg(); + TenantId tenantId = getTenantId(edqsMsg); + ObjectType objectType = ObjectType.valueOf(eventMsg.getObjectType()); + EdqsEventType eventType = EdqsEventType.valueOf(eventMsg.getEventType()); + String key = eventMsg.getKey(); + Long version = eventMsg.hasVersion() ? eventMsg.getVersion() : null; + + if (version != null) { + if (!versionsStore.isNew(key, version)) { + return; + } + } else if (!ObjectType.unversionedTypes.contains(objectType)) { + log.warn("[{}] {} {} doesn't have version", tenantId, objectType, key); + } + if (queue != EdqsQueue.STATE) { + stateService.save(tenantId, objectType, key, eventType, edqsMsg); + } + + EdqsObject object = converter.deserialize(objectType, eventMsg.getData().toByteArray()); + log.debug("[{}] Processing event [{}] [{}] [{}] [{}]", tenantId, objectType, eventType, key, version); + int count = counter.incrementAndGet(); + if (count % 100000 == 0) { + log.info("Processed {} events", count); + } + + EdqsEvent event = EdqsEvent.builder() + .tenantId(tenantId) + .objectType(objectType) + .eventType(eventType) + .object(object) + .build(); + repository.processEvent(event); + } + } + + private TenantId getTenantId(ToEdqsMsg edqsMsg) { + return TenantId.fromUUID(new UUID(edqsMsg.getTenantIdMSB(), edqsMsg.getTenantIdLSB())); + } + + private CustomerId getCustomerId(ToEdqsMsg edqsMsg) { + if (edqsMsg.getCustomerIdMSB() != 0 && edqsMsg.getCustomerIdLSB() != 0) { + return new CustomerId(new UUID(edqsMsg.getCustomerIdMSB(), edqsMsg.getCustomerIdLSB())); + } else { + return null; + } + } + + @PreDestroy + public void destroy() throws InterruptedException { + eventConsumer.stop(); + eventConsumer.awaitStop(); + responseTemplate.stop(); + stateService.stop(); + + consumersExecutor.shutdownNow(); + taskExecutor.shutdownNow(); + scheduler.shutdownNow(); + requestExecutor.shutdownNow(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java new file mode 100644 index 0000000000..be1f0481be --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java @@ -0,0 +1,92 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.processor; + +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.common.errors.RecordTooLargeException; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.edqs.state.EdqsPartitionService; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; + +@Slf4j +public class EdqsProducer { + + private final EdqsQueue queue; + private final EdqsPartitionService partitionService; + private final TopicService topicService; + + private final TbQueueProducer> producer; + + @Builder + public EdqsProducer(EdqsQueue queue, + EdqsPartitionService partitionService, + TopicService topicService, + TbQueueProducer> producer) { + this.queue = queue; + this.partitionService = partitionService; + this.topicService = topicService; + this.producer = producer; + } + + public void send(TenantId tenantId, ObjectType type, String key, ToEdqsMsg msg) { + String topic = topicService.buildTopicName(queue.getTopic()); + TbQueueCallback callback = new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + log.trace("[{}][{}][{}] Published msg to {}: {}", tenantId, type, key, topic, msg); + } + + @Override + public void onFailure(Throwable t) { + if (t instanceof RecordTooLargeException) { + if (!log.isDebugEnabled()) { + log.warn("[{}][{}][{}] Failed to publish msg to {}", tenantId, type, key, topic, t); // not logging the whole message + return; + } + } + log.warn("[{}][{}][{}] Failed to publish msg to {}: {}", tenantId, type, key, topic, msg, t); + } + }; + if (producer instanceof TbKafkaProducerTemplate> kafkaProducer) { + TopicPartitionInfo tpi = TopicPartitionInfo.builder() + .topic(topic) + .partition(partitionService.resolvePartition(tenantId)) + .useInternalPartition(true) + .build(); + kafkaProducer.send(tpi, key, new TbProtoQueueMsg<>(null, msg), callback); // specifying custom key for compaction + } else { + TopicPartitionInfo tpi = TopicPartitionInfo.builder() + .topic(topic) + .build(); + producer.send(tpi, new TbProtoQueueMsg<>(null, msg), callback); + } + } + + public void stop() { + producer.stop(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/DataKey.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/DataKey.java new file mode 100644 index 0000000000..6d9a672e2c --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/DataKey.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import org.thingsboard.server.common.data.query.EntityKeyType; + +public record DataKey(EntityKeyType type, String key, Integer keyId) { + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsCountQuery.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsCountQuery.java new file mode 100644 index 0000000000..9c73c3ed9e --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsCountQuery.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import lombok.Builder; +import org.thingsboard.server.common.data.query.EntityFilter; + +import java.util.List; + +public class EdqsCountQuery extends EdqsQuery { + + @Builder + EdqsCountQuery(EntityFilter entityFilter, boolean hasKeyFilters, List keyFilters) { + super(entityFilter, hasKeyFilters, keyFilters); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsDataQuery.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsDataQuery.java new file mode 100644 index 0000000000..8e118c6e58 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsDataQuery.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.util.CollectionsUtil; + +import java.util.List; + +@EqualsAndHashCode(callSuper = true) +@Getter +public class EdqsDataQuery extends EdqsQuery { + + private final int pageSize; + private final int page; + private final boolean hasTextSearch; + private final String textSearch; + private final boolean defaultSort; + private final DataKey sortKey; + private final EntityDataSortOrder.Direction sortDirection; + private final List entityFields; + private final List latestValues; + + @Builder + public EdqsDataQuery(EntityFilter entityFilter, List keyFilters, + int pageSize, int page, String textSearch, DataKey sortKey, EntityDataSortOrder.Direction sortDirection, + List entityFields, List latestValues) { + super(entityFilter, CollectionsUtil.isNotEmpty(keyFilters), keyFilters); + this.pageSize = pageSize; + this.page = page; + this.hasTextSearch = StringUtils.isNotBlank(textSearch); + this.textSearch = textSearch; + this.defaultSort = EntityKeyType.ENTITY_FIELD.equals(sortKey.type()) && "createdTime".equals(sortKey.key()) && EntityDataSortOrder.Direction.DESC.equals(sortDirection); + this.sortKey = sortKey; + this.sortDirection = sortDirection; + this.entityFields = entityFields; + this.latestValues = latestValues; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsFilter.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsFilter.java new file mode 100644 index 0000000000..67018ebbb8 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsFilter.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.KeyFilterPredicate; + +public record EdqsFilter(DataKey key, EntityKeyValueType valueType, KeyFilterPredicate predicate) { + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsQuery.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsQuery.java new file mode 100644 index 0000000000..aa20c7c306 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsQuery.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import lombok.Data; +import org.thingsboard.server.common.data.query.EntityFilter; + +import java.util.List; + +@Data +public abstract class EdqsQuery { + + private final EntityFilter entityFilter; + private final boolean hasKeyFilters; + private final List keyFilters; + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java new file mode 100644 index 0000000000..026c470ce6 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.edqs.data.EntityData; + +import java.util.UUID; + +@Data +public class SortableEntityData { + + private final EntityData entityData; + private String sortValue; + + public UUID getId(){ + return entityData.getId(); + } + + public EntityId getEntityId() { + return EntityIdFactory.getByTypeAndUuid(entityData.getEntityType(), entityData.getId()); + } +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java new file mode 100644 index 0000000000..b78e49879e --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; + +public abstract class AbstractEntityProfileNameQueryProcessor extends AbstractSimpleQueryProcessor { + + private final Set entityProfileNames; + private final Pattern pattern; + + public AbstractEntityProfileNameQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter, EntityType entityType) { + super(repo, ctx, query, filter, entityType); + entityProfileNames = new HashSet<>(getProfileNames(this.filter)); + pattern = RepositoryUtils.toSqlLikePattern(getEntityNameFilter(filter)); + } + + protected abstract String getEntityNameFilter(T filter); + + protected abstract List getProfileNames(T filter); + + @Override + protected boolean matches(EntityData ed) { + return super.matches(ed) && entityProfileNames.contains(ed.getFields().getType()) + && (pattern == null || pattern.matcher(ed.getFields().getName()).matches()); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java new file mode 100644 index 0000000000..301ead7c63 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.ProfileAwareData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.regex.Pattern; + +public abstract class AbstractEntityProfileQueryProcessor extends AbstractSimpleQueryProcessor { + + private final Set entityProfileIds = new HashSet<>(); + private final Pattern pattern; + + public AbstractEntityProfileQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter, EntityType entityType) { + super(repo, ctx, query, filter, entityType); + var profileNamesSet = new HashSet<>(getProfileNames(this.filter)); + for (EntityData dp : repo.getEntitySet(getProfileEntityType())) { + if (profileNamesSet.contains(dp.getFields().getName())) { + entityProfileIds.add(dp.getId()); + } + } + pattern = RepositoryUtils.toSqlLikePattern(getEntityNameFilter(filter)); + } + + protected abstract String getEntityNameFilter(T filter); + + protected abstract List getProfileNames(T filter); + + protected abstract EntityType getProfileEntityType(); + + @Override + protected boolean matches(EntityData ed) { + ProfileAwareData profileAwareData = (ProfileAwareData) ed; + return super.matches(ed) && entityProfileIds.contains(profileAwareData.getFields().getProfileId()) + && (pattern == null || pattern.matcher(profileAwareData.getFields().getName()).matches()); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntitySearchQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntitySearchQueryProcessor.java new file mode 100644 index 0000000000..b34cd8a459 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntitySearchQueryProcessor.java @@ -0,0 +1,66 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntitySearchQueryFilter; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.Set; +import java.util.UUID; + +public abstract class AbstractEntitySearchQueryProcessor extends AbstractRelationQueryProcessor { + + + public AbstractEntitySearchQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter) { + super(repo, ctx, query, filter); + } + + @Override + public Set getRootEntities() { + return Set.of(filter.getRootEntity().getId()); + } + + @Override + public EntitySearchDirection getDirection() { + return filter.getDirection(); + } + + @Override + public int getMaxLevel() { + return filter.getMaxLevel(); + } + + @Override + public boolean isFetchLastLevelOnly() { + return filter.isFetchLastLevelOnly(); + } + + public abstract EntityType getEntityType(); + + @Override + protected boolean check(RelationInfo relationInfo) { + EntityData target = relationInfo.getTarget(); + return (filter.getRelationType() == null || relationInfo.getType().equals(filter.getRelationType())) && + getEntityType().equals(target.getEntityType()) && super.matches(target); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java new file mode 100644 index 0000000000..e4cded3e3e --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java @@ -0,0 +1,74 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.DataKey; +import org.thingsboard.server.edqs.query.EdqsDataQuery; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.Collection; +import java.util.UUID; +import java.util.function.Consumer; + +import static org.thingsboard.server.edqs.util.RepositoryUtils.checkFilters; +import static org.thingsboard.server.edqs.util.RepositoryUtils.getSortValue; + +public abstract class AbstractQueryProcessor implements EntityQueryProcessor { + + protected final TenantRepo repository; + protected final QueryContext ctx; + protected final EdqsQuery query; + protected final DataKey sortKey; + protected final T filter; + + public AbstractQueryProcessor(TenantRepo repository, QueryContext ctx, EdqsQuery query, T filter) { + this.repository = repository; + this.ctx = ctx; + this.query = query; + this.sortKey = query instanceof EdqsDataQuery dataQuery ? dataQuery.getSortKey() : null; + this.filter = filter; + } + + protected SortableEntityData toSortData(EntityData ed) { + SortableEntityData sortData = new SortableEntityData(ed); + sortData.setSortValue(getSortValue(ed, sortKey)); + return sortData; + } + + protected void process(Collection> entities, Consumer> processor) { + for (EntityData ed : entities) { + if (matches(ed)) { + processor.accept(ed); + } + } + } + + protected static boolean checkCustomerId(UUID customerId, EntityData ed) { + return customerId.equals(ed.getCustomerId()) || (ed.getEntityType() == EntityType.DASHBOARD && + ed.getFields().getAssignedCustomerIds().contains(customerId)); + } + + protected boolean matches(EntityData ed) { + return checkFilters(query, ed); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java new file mode 100644 index 0000000000..8ee7338a4f --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java @@ -0,0 +1,170 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.data.RelationsRepo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import java.util.Set; +import java.util.UUID; + + +public abstract class AbstractRelationQueryProcessor extends AbstractQueryProcessor { + + public static final int MAXIMUM_QUERY_LEVEL = 100; + + public AbstractRelationQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter) { + super(repo, ctx, query, filter); + } + + protected abstract Set getRootEntities(); + + protected abstract EntitySearchDirection getDirection(); + + protected abstract int getMaxLevel(); + + protected abstract boolean isFetchLastLevelOnly(); + + protected boolean isMultiRoot() { + return false; + } + + @Override + public List processQuery() { + var relations = repository.getRelations(RelationTypeGroup.COMMON); + var entities = getEntitiesSet(relations); + if (ctx.isTenantUser()) { + return processTenantQuery(entities); + } else { + return processCustomerQuery(entities); + } + } + + @Override + public long count() { + var relations = repository.getRelations(RelationTypeGroup.COMMON); + var entities = getEntitiesSet(relations); + long result = 0; + + if (ctx.isTenantUser()) { + return entities.size(); + } else { + var customerId = ctx.getCustomerId().getId(); + for (EntityData ed : entities) { + if (checkCustomerId(customerId, ed)) { + result++; + } + } + return result; + } + } + + private List processTenantQuery(Set> entities) { + return entities.stream() + .map(this::toSortData) + .toList(); + } + + private List processCustomerQuery(Set> entities) { + var customerId = ctx.getCustomerId().getId(); + List result = new ArrayList<>(); + for (EntityData ed : entities) { + if (checkCustomerId(customerId, ed)) { + result.add(toSortData(ed)); + } + } + return result; + } + + private Set> getEntitiesSet(RelationsRepo relations) { + Set> result = new HashSet<>(); + Set processed = new HashSet<>(); + Queue tasks = new LinkedList<>(); + int maxLvl = getMaxLevel() == 0 ? MAXIMUM_QUERY_LEVEL : Math.max(1, getMaxLevel()); + for (UUID uuid : getRootEntities()) { + tasks.add(new RelationSearchTask(uuid, 0)); + } + while (!tasks.isEmpty()) { + RelationSearchTask task = tasks.poll(); + if (processed.add(task.entityId)) { + var entityLvl = task.lvl + 1; + Set entities = EntitySearchDirection.FROM.equals(getDirection()) ? relations.getFrom(task.entityId) : relations.getTo(task.entityId); + if (isFetchLastLevelOnly() && entities.isEmpty() && task.previous != null && check(task.previous)) { + result.add(task.previous.getTarget()); + } + for (RelationInfo relationInfo : entities) { + var entity = relationInfo.getTarget(); + if (entity.isEmpty()) { + continue; + } + var entityId = entity.getId(); + if (isFetchLastLevelOnly()) { + if (entityLvl < maxLvl) { + tasks.add(new RelationSearchTask(entityId, entityLvl, relationInfo)); + } else if (entityLvl == maxLvl) { + if (check(relationInfo)) { + if (isMultiRoot()) { + ctx.getRelatedParentIdMap().put(entity.getId(), task.entityId); + } + result.add(entity); + } + } + } else { + if (check(relationInfo)) { + if (isMultiRoot()) { + ctx.getRelatedParentIdMap().put(entity.getId(), task.entityId); + } + result.add(entity); + } + if (entityLvl < maxLvl) { + tasks.add(new RelationSearchTask(entityId, entityLvl)); + } + } + } + } + } + return result; + } + + protected abstract boolean check(RelationInfo relationInfo); + + @RequiredArgsConstructor + private static class RelationSearchTask { + private final UUID entityId; + private final int lvl; + private final RelationInfo previous; + + public RelationSearchTask(UUID entityId, int lvl) { + this(entityId, lvl, null); + } + + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSimpleQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSimpleQueryProcessor.java new file mode 100644 index 0000000000..aab1e83879 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSimpleQueryProcessor.java @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.edqs.data.CustomerData; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.UUID; +import java.util.function.Consumer; + +public abstract class AbstractSimpleQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + private final EntityType entityType; + + public AbstractSimpleQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter, EntityType entityType) { + super(repo, ctx, query, filter); + this.entityType = entityType; + } + + @Override + protected void processCustomerQuery(UUID customerId, Consumer> processor) { + var customerData = (CustomerData) repository.getEntityMap(EntityType.CUSTOMER).get(customerId); + if (customerData != null) { + process(customerData.getEntities(entityType), processor); + } + } + + @Override + protected void processAll(Consumer> processor) { + process(repository.getEntitySet(entityType), processor); + } + + @Override + protected int getProbableResultSize() { + return 1024; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSingleEntityTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSingleEntityTypeQueryProcessor.java new file mode 100644 index 0000000000..1723ee5f5b --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSingleEntityTypeQueryProcessor.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; + +public abstract class AbstractSingleEntityTypeQueryProcessor extends AbstractQueryProcessor { + + public AbstractSingleEntityTypeQueryProcessor(TenantRepo repository, QueryContext ctx, EdqsQuery query, T filter) { + super(repository, ctx, query, filter); + } + + @Override + public List processQuery() { + if (ctx.isTenantUser()) { + return processTenantQuery(); + } else { + return processCustomerQuery(ctx.getCustomerId().getId()); + } + } + + @Override + public long count() { + AtomicLong result = new AtomicLong(); + Consumer> counter = ed -> result.incrementAndGet(); + + if (ctx.isIgnorePermissionCheck()) { + processAll(counter); + } else if (ctx.isTenantUser()) { + processAll(counter); + } else { + processCustomerQuery(ctx.getCustomerId().getId(), counter); + } + return result.get(); + } + + protected List processTenantQuery() { + List result = new ArrayList<>(getProbableResultSize()); + processAll(ed -> { + result.add(toSortData(ed)); + }); + return result; + } + + protected List processCustomerQuery(UUID customerId) { + List result = new ArrayList<>(getProbableResultSize()); + processCustomerQuery(customerId, ed -> { + result.add(toSortData(ed)); + }); + return result; + } + + protected abstract void processCustomerQuery(UUID customerId, Consumer> processor); + + protected abstract void processAll(Consumer> processor); + + protected abstract int getProbableResultSize(); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java new file mode 100644 index 0000000000..44370292e4 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.ApiUsageStateFields; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.ApiUsageStateFilter; +import org.thingsboard.server.edqs.data.CustomerData; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.UUID; +import java.util.function.Consumer; + +public class ApiUsageStateQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + public ApiUsageStateQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (ApiUsageStateFilter) query.getEntityFilter()); + } + + @Override + protected void processCustomerQuery(UUID customerId, Consumer> processor) { + CustomerData customerData = (CustomerData) repository.getEntityMap(EntityType.CUSTOMER).get(customerId); + if (customerData != null) { + process(customerData.getEntities(EntityType.API_USAGE_STATE), processor); + } + } + + @Override + protected void processAll(Consumer> processor) { + process(repository.getEntitySet(EntityType.API_USAGE_STATE), processor); + } + + @Override + protected boolean matches(EntityData ed) { + ApiUsageStateFields entityFields = (ApiUsageStateFields) ed.getFields(); + return super.matches(ed) && (filter.getCustomerId() == null || filter.getCustomerId().equals(entityFields.getEntityId())); + } + + @Override + protected int getProbableResultSize() { + return 1; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetSearchQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetSearchQueryProcessor.java new file mode 100644 index 0000000000..2eff8f6d15 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetSearchQueryProcessor.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; +import org.thingsboard.server.common.data.util.CollectionsUtil; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.ProfileAwareData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; + +public class AssetSearchQueryProcessor extends AbstractEntitySearchQueryProcessor { + + private final Set entityProfileIds = new HashSet<>(); + + public AssetSearchQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (AssetSearchQueryFilter) query.getEntityFilter()); + if (CollectionsUtil.isNotEmpty(filter.getAssetTypes())) { + var profileNamesSet = new HashSet<>(this.filter.getAssetTypes()); + for (EntityData dp : repo.getEntitySet(EntityType.ASSET_PROFILE)) { + if (profileNamesSet.contains(dp.getFields().getName())) { + entityProfileIds.add(dp.getId()); + } + } + } + } + + @Override + public EntityType getEntityType() { + return EntityType.ASSET; + } + + @Override + protected boolean check(RelationInfo relationInfo) { + return super.check(relationInfo) && + (entityProfileIds.isEmpty() || entityProfileIds.contains(((ProfileAwareData) relationInfo.getTarget()).getFields().getProfileId())); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetTypeQueryProcessor.java new file mode 100644 index 0000000000..cee5edfa9b --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetTypeQueryProcessor.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.AssetTypeFilter; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.List; + +public class AssetTypeQueryProcessor extends AbstractEntityProfileQueryProcessor { + + public AssetTypeQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (AssetTypeFilter) query.getEntityFilter(), EntityType.ASSET); + } + + @Override + protected String getEntityNameFilter(AssetTypeFilter filter) { + return filter.getAssetNameFilter(); + } + + @Override + protected List getProfileNames(AssetTypeFilter filter) { + return filter.getAssetTypes(); + } + + @Override + protected EntityType getProfileEntityType() { + return EntityType.ASSET_PROFILE; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceSearchQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceSearchQueryProcessor.java new file mode 100644 index 0000000000..3e53c0815f --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceSearchQueryProcessor.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.DeviceSearchQueryFilter; +import org.thingsboard.server.common.data.util.CollectionsUtil; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.ProfileAwareData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; + +public class DeviceSearchQueryProcessor extends AbstractEntitySearchQueryProcessor { + + private final Set entityProfileIds = new HashSet<>(); + + public DeviceSearchQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (DeviceSearchQueryFilter) query.getEntityFilter()); + if (CollectionsUtil.isNotEmpty(filter.getDeviceTypes())) { + var profileNamesSet = new HashSet<>(this.filter.getDeviceTypes()); + for (EntityData dp : repo.getEntitySet(EntityType.DEVICE_PROFILE)) { + if (profileNamesSet.contains(dp.getFields().getName())) { + entityProfileIds.add(dp.getId()); + } + } + } + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE; + } + + @Override + protected boolean check(RelationInfo relationInfo) { + return super.check(relationInfo) && + (entityProfileIds.isEmpty() || entityProfileIds.contains(((ProfileAwareData) relationInfo.getTarget()).getFields().getProfileId())); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceTypeQueryProcessor.java new file mode 100644 index 0000000000..44eaf3e74a --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceTypeQueryProcessor.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.DeviceTypeFilter; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.List; + +public class DeviceTypeQueryProcessor extends AbstractEntityProfileQueryProcessor { + + public DeviceTypeQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (DeviceTypeFilter) query.getEntityFilter(), EntityType.DEVICE); + } + + @Override + protected String getEntityNameFilter(DeviceTypeFilter filter) { + return filter.getDeviceNameFilter(); + } + + @Override + protected List getProfileNames(DeviceTypeFilter filter) { + return filter.getDeviceTypes(); + } + + @Override + protected EntityType getProfileEntityType() { + return EntityType.DEVICE_PROFILE; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeQueryProcessor.java new file mode 100644 index 0000000000..965d3e09ca --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeQueryProcessor.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EdgeTypeFilter; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.List; + +public class EdgeTypeQueryProcessor extends AbstractEntityProfileNameQueryProcessor { + + public EdgeTypeQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EdgeTypeFilter) query.getEntityFilter(), EntityType.EDGE); + } + + @Override + protected String getEntityNameFilter(EdgeTypeFilter filter) { + return filter.getEdgeNameFilter(); + } + + @Override + protected List getProfileNames(EdgeTypeFilter filter) { + return filter.getEdgeTypes(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeSearchQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeSearchQueryProcessor.java new file mode 100644 index 0000000000..e9e174505b --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeSearchQueryProcessor.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EdgeSearchQueryFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +public class EdgeTypeSearchQueryProcessor extends AbstractEntitySearchQueryProcessor { + + public EdgeTypeSearchQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EdgeSearchQueryFilter) query.getEntityFilter()); + } + + @Override + public EntityType getEntityType() { + return EntityType.EDGE; + } + + @Override + protected boolean check(RelationInfo relationInfo) { + EntityData ed = relationInfo.getTarget(); + return super.check(relationInfo) && + (filter.getEdgeTypes() == null || filter.getEdgeTypes().contains(ed.getFields().getType())); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java new file mode 100644 index 0000000000..3a1eebf1e9 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java @@ -0,0 +1,66 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.Set; +import java.util.UUID; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +public class EntityListQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + private final EntityType entityType; + private final Set entityIds; + + public EntityListQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityListFilter) query.getEntityFilter()); + this.entityType = filter.getEntityType(); + this.entityIds = filter.getEntityList().stream().map(UUID::fromString).collect(Collectors.toSet()); + } + + @Override + protected void processCustomerQuery(UUID customerId, Consumer> processor) { + processAll(ed -> { + if (checkCustomerId(customerId, ed)) { + processor.accept(ed); + } + }); + } + + @Override + protected void processAll(Consumer> processor) { + var map = repository.getEntityMap(entityType); + for (UUID entityId : entityIds) { + EntityData ed = map.get(entityId); + if (matches(ed)) { + processor.accept(ed); + } + } + } + + @Override + protected int getProbableResultSize() { + return entityIds.size(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java new file mode 100644 index 0000000000..ec88db4d0f --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityNameFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.regex.Pattern; + +public class EntityNameQueryProcessor extends AbstractSimpleQueryProcessor { + + private final Pattern pattern; + + public EntityNameQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityNameFilter) query.getEntityFilter(), ((EntityNameFilter) query.getEntityFilter()).getEntityType()); + pattern = RepositoryUtils.toSqlLikePattern(filter.getEntityNameFilter()); + } + + @Override + protected boolean matches(EntityData ed) { + return ed.getFields() != null && (pattern == null || pattern.matcher(ed.getFields().getName()).matches()); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessor.java new file mode 100644 index 0000000000..ad3fc0e7b6 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessor.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.edqs.query.SortableEntityData; + +import java.util.List; + +public interface EntityQueryProcessor { + + List processQuery(); + + long count(); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessorFactory.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessorFactory.java new file mode 100644 index 0000000000..12fc863566 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessorFactory.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +public class EntityQueryProcessorFactory { + + public static EntityQueryProcessor create(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + return switch (query.getEntityFilter().getType()) { + case SINGLE_ENTITY -> new SingleEntityQueryProcessor(repo, ctx, query); + case ENTITY_LIST -> new EntityListQueryProcessor(repo, ctx, query); + case ENTITY_NAME -> new EntityNameQueryProcessor(repo, ctx, query); + case ENTITY_TYPE -> new EntityTypeQueryProcessor(repo, ctx, query); + case DEVICE_TYPE -> new DeviceTypeQueryProcessor(repo, ctx, query); + case ASSET_TYPE -> new AssetTypeQueryProcessor(repo, ctx, query); + case ENTITY_VIEW_TYPE -> new EntityViewTypeQueryProcessor(repo, ctx, query); + case EDGE_TYPE -> new EdgeTypeQueryProcessor(repo, ctx, query); + case RELATIONS_QUERY -> new RelationQueryProcessor(repo, ctx, query); + case API_USAGE_STATE -> new ApiUsageStateQueryProcessor(repo, ctx, query); + case ASSET_SEARCH_QUERY -> new AssetSearchQueryProcessor(repo, ctx, query); + case DEVICE_SEARCH_QUERY -> new DeviceSearchQueryProcessor(repo, ctx, query); + case ENTITY_VIEW_SEARCH_QUERY -> new EntityViewSearchQueryProcessor(repo, ctx, query); + case EDGE_SEARCH_QUERY -> new EdgeTypeSearchQueryProcessor(repo, ctx, query); + default -> throw new RuntimeException("Not Implemented!"); + }; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityTypeQueryProcessor.java new file mode 100644 index 0000000000..6f6ec3d007 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityTypeQueryProcessor.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +public class EntityTypeQueryProcessor extends AbstractSimpleQueryProcessor { + + public EntityTypeQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityTypeFilter) query.getEntityFilter(), ((EntityTypeFilter) query.getEntityFilter()).getEntityType()); + } + + @Override + protected boolean matches(EntityData ed) { + return super.matches(ed); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewSearchQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewSearchQueryProcessor.java new file mode 100644 index 0000000000..80f56c2169 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewSearchQueryProcessor.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityViewSearchQueryFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +public class EntityViewSearchQueryProcessor extends AbstractEntitySearchQueryProcessor { + + public EntityViewSearchQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityViewSearchQueryFilter) query.getEntityFilter()); + } + + @Override + public EntityType getEntityType() { + return EntityType.ENTITY_VIEW; + } + + @Override + protected boolean check(RelationInfo relationInfo) { + EntityData ed = relationInfo.getTarget(); + return super.check(relationInfo) && + (filter.getEntityViewTypes() == null || filter.getEntityViewTypes().contains(ed.getFields().getType())); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewTypeQueryProcessor.java new file mode 100644 index 0000000000..2ce4e8616b --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewTypeQueryProcessor.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityViewTypeFilter; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.List; + +public class EntityViewTypeQueryProcessor extends AbstractEntityProfileNameQueryProcessor { + + public EntityViewTypeQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityViewTypeFilter) query.getEntityFilter(), EntityType.ENTITY_VIEW); + } + + @Override + protected String getEntityNameFilter(EntityViewTypeFilter filter) { + return filter.getEntityViewNameFilter(); + } + + @Override + protected List getProfileNames(EntityViewTypeFilter filter) { + return filter.getEntityViewTypes(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryProcessor.java new file mode 100644 index 0000000000..d4928a5832 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryProcessor.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +public class RelationQueryProcessor extends AbstractRelationQueryProcessor { + + private final boolean hasFilters; + + public RelationQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (RelationsQueryFilter) query.getEntityFilter()); + this.hasFilters = filter.getFilters() != null && !filter.getFilters().isEmpty(); + } + + @Override + public Set getRootEntities() { + if (filter.isMultiRoot()) { + return filter.getMultiRootEntityIds().stream().map(UUID::fromString).collect(Collectors.toSet()); + } else { + return Set.of(filter.getRootEntity().getId()); + } + } + + @Override + public EntitySearchDirection getDirection() { + return filter.getDirection(); + } + + @Override + public int getMaxLevel() { + return filter.getMaxLevel(); + } + + @Override + public boolean isMultiRoot() { + return filter.isMultiRoot(); + } + + @Override + public boolean isFetchLastLevelOnly() { + return filter.isFetchLastLevelOnly(); + } + + @Override + protected boolean check(RelationInfo relationInfo) { + if (hasFilters) { + for (var f : filter.getFilters()) { + if (((!filter.isNegate() && !f.isNegate()) || (filter.isNegate() && f.isNegate())) == f.getRelationType().equals(relationInfo.getType())) { + if (f.getEntityTypes() == null || f.getEntityTypes().isEmpty() + || f.getEntityTypes().contains(relationInfo.getTarget().getEntityType())) { + return super.matches(relationInfo.getTarget()); + } + } + } + return false; + } else { + return super.matches(relationInfo.getTarget()); + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java new file mode 100644 index 0000000000..febf228699 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.SingleEntityFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.UUID; +import java.util.function.Consumer; + +public class SingleEntityQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + private final EntityType entityType; + private final UUID entityId; + + public SingleEntityQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (SingleEntityFilter) query.getEntityFilter()); + this.entityType = filter.getSingleEntity().getEntityType(); + this.entityId = filter.getSingleEntity().getId(); + } + + @Override + protected void processCustomerQuery(UUID customerId, Consumer> processor) { + processAll(ed -> { + if (checkCustomerId(customerId, ed)) { + processor.accept(ed); + } + }); + } + + @Override + protected void processAll(Consumer> processor) { + EntityData ed = repository.getEntityMap(entityType).get(entityId); + if (matches(ed)) { + processor.accept(ed); + } + } + + @Override + protected int getProbableResultSize() { + return 1; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java new file mode 100644 index 0000000000..1deaca83a7 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEvent; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.edqs.stats.EdqsStatsService; +import org.thingsboard.server.queue.edqs.EdqsComponent; + +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Predicate; + +@EdqsComponent +@AllArgsConstructor +@Service +@Slf4j +public class DefaultEdqsRepository implements EdqsRepository { + + private final static ConcurrentMap repos = new ConcurrentHashMap<>(); + private final Optional statsService; + + public TenantRepo get(TenantId tenantId) { + return repos.computeIfAbsent(tenantId, id -> new TenantRepo(id, statsService)); + } + + @Override + public void processEvent(EdqsEvent event) { + if (event.getEventType() == EdqsEventType.DELETED && event.getObjectType() == ObjectType.TENANT) { + log.info("Tenant {} deleted", event.getTenantId()); + repos.remove(event.getTenantId()); + } else { + get(event.getTenantId()).processEvent(event); + } + } + + @Override + public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query, boolean ignorePermissionCheck) { + long startNs = System.nanoTime(); + long result = get(tenantId).countEntitiesByQuery(customerId, query, ignorePermissionCheck); + double timingMs = (double) (System.nanoTime() - startNs) / 1000_000; + log.info("countEntitiesByQuery done in {} ms", timingMs); + return result; + } + + @Override + public PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, + EntityDataQuery query, boolean ignorePermissionCheck) { + long startNs = System.nanoTime(); + var result = get(tenantId).findEntityDataByQuery(customerId, query, ignorePermissionCheck); + double timingMs = (double) (System.nanoTime() - startNs) / 1000_000; + log.info("findEntityDataByQuery done in {} ms", timingMs); + return result; + } + + @Override + public void clearIf(Predicate predicate) { + repos.keySet().removeIf(predicate); + } + + @Override + public void clear() { + repos.clear(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/EdqsRepository.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/EdqsRepository.java new file mode 100644 index 0000000000..3d9f2ab8df --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/EdqsRepository.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.thingsboard.server.common.data.edqs.EdqsEvent; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; + +import java.util.function.Predicate; + +public interface EdqsRepository { + + void processEvent(EdqsEvent event); + + long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query, boolean ignorePermissionCheck); + + PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query, boolean ignorePermissionCheck); + + void clearIf(Predicate predicate); + + void clear(); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/KeyDictionary.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/KeyDictionary.java new file mode 100644 index 0000000000..0a79b300a9 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/KeyDictionary.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +public class KeyDictionary { + + private static final ConcurrentMap keyToIdDict = new ConcurrentHashMap<>(); + private static final ConcurrentMap idToKeyDict = new ConcurrentHashMap<>(); + private static final AtomicInteger keySeq = new AtomicInteger(); + + public static Integer get(String key) { + return keyToIdDict.computeIfAbsent(key, __ -> { + int keyId = keySeq.incrementAndGet(); + idToKeyDict.put(keyId, key); + return keyId; + }); + } + + public static String get(Integer keyId) { + return idToKeyDict.get(keyId); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java new file mode 100644 index 0000000000..0c55bc50dd --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java @@ -0,0 +1,439 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.AttributeKv; +import org.thingsboard.server.common.data.edqs.DataPoint; +import org.thingsboard.server.common.data.edqs.EdqsEvent; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.Entity; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.edqs.fields.AssetFields; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.TsValue; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.data.ApiUsageStateData; +import org.thingsboard.server.edqs.data.AssetData; +import org.thingsboard.server.edqs.data.CustomerData; +import org.thingsboard.server.edqs.data.DeviceData; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.EntityProfileData; +import org.thingsboard.server.edqs.data.GenericData; +import org.thingsboard.server.edqs.data.RelationsRepo; +import org.thingsboard.server.edqs.data.TenantData; +import org.thingsboard.server.edqs.query.EdqsDataQuery; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.query.processor.EntityQueryProcessor; +import org.thingsboard.server.edqs.query.processor.EntityQueryProcessorFactory; +import org.thingsboard.server.edqs.stats.EdqsStatsService; +import org.thingsboard.server.edqs.util.RepositoryUtils; +import org.thingsboard.server.edqs.util.TbStringPool; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +import static org.thingsboard.server.edqs.util.RepositoryUtils.SORT_ASC; +import static org.thingsboard.server.edqs.util.RepositoryUtils.SORT_DESC; +import static org.thingsboard.server.edqs.util.RepositoryUtils.resolveEntityType; + +@Slf4j +public class TenantRepo { + + public static final Comparator> CREATED_TIME_COMPARATOR = Comparator.comparingLong(ed -> ed.getFields().getCreatedTime()); + public static final Comparator> CREATED_TIME_AND_ID_COMPARATOR = CREATED_TIME_COMPARATOR + .thenComparing(EntityData::getId); + public static final Comparator> CREATED_TIME_AND_ID_DESC_COMPARATOR = CREATED_TIME_AND_ID_COMPARATOR.reversed(); + + private final ConcurrentMap>> entitySetByType = new ConcurrentHashMap<>(); + private final ConcurrentMap>> entityMapByType = new ConcurrentHashMap<>(); + private final ConcurrentMap relations = new ConcurrentHashMap<>(); + + private final Lock entityUpdateLock = new ReentrantLock(); + + private final TenantId tenantId; + private final Optional edqsStatsService; + + public TenantRepo(TenantId tenantId, Optional edqsStatsService) { + this.tenantId = tenantId; + this.edqsStatsService = edqsStatsService; + } + + public void processEvent(EdqsEvent event) { + EdqsObject edqsObject = event.getObject(); + log.trace("[{}] Processing event: {}", tenantId, event); + if (event.getEventType() == EdqsEventType.UPDATED) { + addOrUpdate(edqsObject); + } else if (event.getEventType() == EdqsEventType.DELETED) { + remove(edqsObject); + } + } + + public void addOrUpdate(EdqsObject object) { + if (object instanceof EntityRelation relation) { + addOrUpdateRelation(relation); + } else if (object instanceof AttributeKv attributeKv) { + addOrUpdateAttribute(attributeKv); + } else if (object instanceof LatestTsKv latestTsKv) { + addOrUpdateLatestKv(latestTsKv); + } else if (object instanceof Entity entity) { + addOrUpdateEntity(entity); + } + } + + public void remove(EdqsObject object) { + if (object instanceof EntityRelation relation) { + removeRelation(relation); + } else if (object instanceof AttributeKv attributeKv) { + removeAttribute(attributeKv); + } else if (object instanceof LatestTsKv latestTsKv) { + removeLatestKv(latestTsKv); + } else if (object instanceof Entity entity) { + removeEntity(entity); + } + } + + private void addOrUpdateRelation(EntityRelation entity) { + entityUpdateLock.lock(); + try { + if (RelationTypeGroup.COMMON.equals(entity.getTypeGroup())) { + RelationsRepo repo = relations.computeIfAbsent(entity.getTypeGroup(), tg -> new RelationsRepo()); + EntityData from = getOrCreate(entity.getFrom()); + EntityData to = getOrCreate(entity.getTo()); + boolean added = repo.add(from, to, TbStringPool.intern(entity.getType())); + if (added) { + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.RELATION, EdqsEventType.UPDATED)); + } + } else if (RelationTypeGroup.DASHBOARD.equals(entity.getTypeGroup())) { + if (EntityRelation.CONTAINS_TYPE.equals(entity.getType()) && entity.getFrom().getEntityType() == EntityType.CUSTOMER) { + ((CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(entity.getFrom().getId(), CustomerData::new)) + .addOrUpdate(getEntityMap(EntityType.DASHBOARD).get(entity.getTo().getId())); + } + } + } finally { + entityUpdateLock.unlock(); + } + } + + private void removeRelation(EntityRelation entityRelation) { + if (RelationTypeGroup.COMMON.equals(entityRelation.getTypeGroup())) { + RelationsRepo relationsRepo = relations.get(entityRelation.getTypeGroup()); + if (relationsRepo != null) { + boolean removed = relationsRepo.remove(entityRelation.getFrom().getId(), entityRelation.getTo().getId(), entityRelation.getType()); + if (removed) { + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.RELATION, EdqsEventType.DELETED)); + } + } + } else if (RelationTypeGroup.DASHBOARD.equals(entityRelation.getTypeGroup())) { + if (EntityRelation.CONTAINS_TYPE.equals(entityRelation.getType()) && entityRelation.getFrom().getEntityType() == EntityType.CUSTOMER) { + ((CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(entityRelation.getFrom().getId(), CustomerData::new)) + .remove(getEntityMap(EntityType.DASHBOARD).get(entityRelation.getTo().getId())); + } + } + } + + private void addOrUpdateEntity(Entity entity) { + entityUpdateLock.lock(); + try { + log.trace("[{}] addOrUpdateEntity: {}", tenantId, entity); + EntityFields fields = entity.getFields(); + UUID entityId = fields.getId(); + EntityType entityType = entity.getType(); + + EntityData entityData = getOrCreate(entityType, entityId); + processFields(fields); + EntityFields oldFields = entityData.getFields(); + entityData.setFields(fields); + if (oldFields == null) { + getEntitySet(entityType).add(entityData); + } + + UUID newCustomerId = fields.getCustomerId(); + UUID oldCustomerId = entityData.getCustomerId(); + entityData.setCustomerId(newCustomerId); + if (entityIdMismatch(oldCustomerId, newCustomerId)) { + if (oldCustomerId != null) { + CustomerData old = (CustomerData) getEntityMap(EntityType.CUSTOMER).get(oldCustomerId); + if (old != null) { + old.remove(entityData); + } + } + if (newCustomerId != null) { + CustomerData newData = (CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(newCustomerId, CustomerData::new); + newData.addOrUpdate(entityData); + } + } + } finally { + entityUpdateLock.unlock(); + } + } + + public void removeEntity(Entity entity) { + entityUpdateLock.lock(); + try { + UUID entityId = entity.getFields().getId(); + EntityType entityType = entity.getType(); + EntityData removed = getEntityMap(entityType).remove(entityId); + if (removed != null) { + if (removed.getFields() != null) { + getEntitySet(entityType).remove(removed); + } + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.DELETED)); + UUID customerId = removed.getCustomerId(); + if (customerId != null) { + CustomerData customerData = (CustomerData) getEntityMap(EntityType.CUSTOMER).get(customerId); + if (customerData != null) { + customerData.remove(removed); + } + } + } + } finally { + entityUpdateLock.unlock(); + } + } + + public void addOrUpdateAttribute(AttributeKv attributeKv) { + var entityData = getOrCreate(attributeKv.getEntityId()); + if (entityData != null) { + Integer keyId = KeyDictionary.get(attributeKv.getKey()); + boolean added = entityData.putAttr(keyId, attributeKv.getScope(), attributeKv.getDataPoint()); + if (added) { + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.ATTRIBUTE_KV, EdqsEventType.UPDATED)); + } + } + } + + private void removeAttribute(AttributeKv attributeKv) { + var entityData = get(attributeKv.getEntityId()); + if (entityData != null) { + boolean removed = entityData.removeAttr(KeyDictionary.get(attributeKv.getKey()), attributeKv.getScope()); + if (removed) { + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.ATTRIBUTE_KV, EdqsEventType.DELETED)); + } + } + } + + public void addOrUpdateLatestKv(LatestTsKv latestTsKv) { + var entityData = getOrCreate(latestTsKv.getEntityId()); + if (entityData != null) { + Integer keyId = KeyDictionary.get(latestTsKv.getKey()); + boolean added = entityData.putTs(keyId, latestTsKv.getDataPoint()); + if (added) { + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.LATEST_TS_KV, EdqsEventType.UPDATED)); + } + } + } + + private void removeLatestKv(LatestTsKv latestTsKv) { + var entityData = get(latestTsKv.getEntityId()); + if (entityData != null) { + boolean removed = entityData.removeTs(KeyDictionary.get(latestTsKv.getKey())); + if (removed) { + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.LATEST_TS_KV, EdqsEventType.DELETED)); + } + } + } + + public void processFields(EntityFields fields) { + if (fields instanceof AssetFields assetFields) { + assetFields.setType(TbStringPool.intern(assetFields.getType())); + } + } + + public ConcurrentMap> getEntityMap(EntityType entityType) { + return entityMapByType.computeIfAbsent(entityType, et -> new ConcurrentHashMap<>()); + } + + //TODO: automatically remove entities that has nothing except the ID. + private EntityData getOrCreate(EntityId entityId) { + return getOrCreate(entityId.getEntityType(), entityId.getId()); + } + + private EntityData getOrCreate(EntityType entityType, UUID entityId) { + return getEntityMap(entityType).computeIfAbsent(entityId, id -> { + log.debug("[{}] Adding {} {}", tenantId, entityType, id); + EntityData entityData = constructEntityData(entityType, entityId); + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.UPDATED)); + return entityData; + }); + } + + private EntityData get(EntityId entityId) { + return getEntityMap(entityId.getEntityType()).get(entityId.getId()); + } + + private EntityData constructEntityData(EntityType entityType, UUID id) { + EntityData entityData = switch (entityType) { + case DEVICE -> new DeviceData(id); + case ASSET -> new AssetData(id); + case DEVICE_PROFILE, ASSET_PROFILE -> new EntityProfileData(id, entityType); + case CUSTOMER -> new CustomerData(id); + case TENANT -> new TenantData(id); + case API_USAGE_STATE -> new ApiUsageStateData(id); + default -> new GenericData(entityType, id); + }; + entityData.setRepo(this); + return entityData; + } + + private static boolean entityIdMismatch(UUID oldOrNull, UUID newOrNull) { + if (oldOrNull == null) { + return newOrNull != null; + } else { + return !oldOrNull.equals(newOrNull); + } + } + + public Set> getEntitySet(EntityType entityType) { + return entitySetByType.computeIfAbsent(entityType, et -> new ConcurrentSkipListSet<>(CREATED_TIME_AND_ID_DESC_COMPARATOR)); + } + + public PageData findEntityDataByQuery(CustomerId customerId, EntityDataQuery oldQuery, boolean ignorePermissionCheck) { + EdqsDataQuery query = RepositoryUtils.toNewQuery(oldQuery); + log.info("[{}][{}] findEntityDataByQuery: {}", tenantId, customerId, query); + QueryContext ctx = buildContext(customerId, query.getEntityFilter(), ignorePermissionCheck); + EntityQueryProcessor queryProcessor = EntityQueryProcessorFactory.create(this, ctx, query); + return sortAndConvert(query, queryProcessor.processQuery(), ctx); + } + + public long countEntitiesByQuery(CustomerId customerId, EntityCountQuery oldQuery, boolean ignorePermissionCheck) { + EdqsQuery query = RepositoryUtils.toNewQuery(oldQuery); + log.info("[{}][{}] countEntitiesByQuery: {}", tenantId, customerId, query); + QueryContext ctx = buildContext(customerId, query.getEntityFilter(), ignorePermissionCheck); + EntityQueryProcessor queryProcessor = EntityQueryProcessorFactory.create(this, ctx, query); + return queryProcessor.count(); + } + + private PageData sortAndConvert(EdqsDataQuery query, List data, QueryContext ctx) { + int totalSize = data.size(); + int totalPages = (int) Math.ceil((float) totalSize / query.getPageSize()); + int offset = query.getPage() * query.getPageSize(); + if (offset > totalSize) { + return new PageData<>(Collections.emptyList(), totalPages, totalSize, false); + } else { + Comparator comparator = EntityDataSortOrder.Direction.ASC.equals(query.getSortDirection()) ? SORT_ASC : SORT_DESC; + long startTs = System.nanoTime(); +// IMPLEMENTATION THAT IS BASED ON PRIORITY_QUEUE +// var requiredSize = Math.min(offset + query.getPageSize(), totalSize); +// PriorityQueue topN = new PriorityQueue<>(requiredSize, comparator.reversed()); +// for (SortableEntityData item : data) { +// topN.add(item); +// if (topN.size() > requiredSize) { +// topN.poll(); +// } +// } +// List result = new ArrayList<>(topN); +// Collections.reverse(result); +// result = result.subList(offset, requiredSize); +// IMPLEMENTATION THAT IS BASED ON TREE SET (For offset + query.getPageSize() << totalSize) + var requiredSize = Math.min(offset + query.getPageSize(), totalSize); + TreeSet topNSet = new TreeSet<>(comparator); + for (SortableEntityData sp : data) { + topNSet.add(sp); + if (topNSet.size() > requiredSize) { + topNSet.pollLast(); + } + } + var result = topNSet.stream().skip(offset).limit(query.getPageSize()).collect(Collectors.toList()); +// IMPLEMENTATION THAT IS BASED ON TIM SORT (For offset + query.getPageSize() > totalSize / 2) +// data.sort(comparator); +// var result = data.subList(offset, endIndex); + log.trace("EDQ Sorted in {}", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTs)); + return new PageData<>(toQueryResult(result, query, ctx), totalPages, totalSize, totalSize > requiredSize); + } + } + + private List toQueryResult(List data, EdqsDataQuery query, QueryContext ctx) { + long ts = System.currentTimeMillis(); + List results = new ArrayList<>(data.size()); + for (SortableEntityData entityData : data) { + Map> latest = new HashMap<>(); + for (var key : query.getEntityFields()) { + DataPoint dp = entityData.getEntityData().getDataPoint(key, ctx); + TsValue v = RepositoryUtils.toTsValue(ts, dp); + latest.computeIfAbsent(EntityKeyType.ENTITY_FIELD, t -> new HashMap<>()).put(key.key(), v); + } + for (var key : query.getLatestValues()) { + DataPoint dp = entityData.getEntityData().getDataPoint(key, ctx); + TsValue v = RepositoryUtils.toTsValue(ts, dp); + latest.computeIfAbsent(key.type(), t -> new HashMap<>()).put(KeyDictionary.get(key.keyId()), v); + } + + results.add(new QueryResult(entityData.getEntityId(), latest)); + } + return results; + } + + private QueryContext buildContext(CustomerId customerId, EntityFilter filter, boolean ignorePermissionCheck) { + return new QueryContext(tenantId, customerId, resolveEntityType(filter), ignorePermissionCheck); + } + + public TenantId getTenantId() { + return tenantId; + } + + + public RelationsRepo getRelations(RelationTypeGroup relationTypeGroup) { + return relations.computeIfAbsent(relationTypeGroup, type -> new RelationsRepo()); + } + + public String getOwnerName(EntityId ownerId) { + if (ownerId == null || (EntityType.CUSTOMER.equals(ownerId.getEntityType()) && CustomerId.NULL_UUID.equals(ownerId.getId()))) { + ownerId = tenantId; + } + return getEntityName(ownerId); + } + + private String getEntityName(EntityId entityId) { + EntityType entityType = entityId.getEntityType(); + return switch (entityType) { + case CUSTOMER, TENANT -> getEntityMap(entityType).get(entityId.getId()).getFields().getName(); + default -> throw new RuntimeException("Unsupported entity type: " + entityType); + }; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsPartitionService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsPartitionService.java new file mode 100644 index 0000000000..94e9437650 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsPartitionService.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.state; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.discovery.HashPartitionService; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsConfig.EdqsPartitioningStrategy; + +@Service +@RequiredArgsConstructor +public class EdqsPartitionService { + + private final HashPartitionService hashPartitionService; + private final EdqsConfig edqsConfig; + + public Integer resolvePartition(TenantId tenantId) { + if (edqsConfig.getPartitioningStrategy() == EdqsPartitioningStrategy.TENANT) { + return hashPartitionService.resolvePartitionIndex(tenantId.getId(), edqsConfig.getPartitions()); + } else { + return null; + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java new file mode 100644 index 0000000000..ee7b058a8a --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.state; + +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; + +import java.util.Set; + +public interface EdqsStateService { + + void init(PartitionedQueueConsumerManager> eventConsumer); + + void process(Set partitions); + + void save(TenantId tenantId, ObjectType type, String key, EdqsEventType eventType, ToEdqsMsg msg); + + boolean isReady(); + + void stop(); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java new file mode 100644 index 0000000000..80b6eebf5c --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java @@ -0,0 +1,187 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.state; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.edqs.processor.EdqsProcessor; +import org.thingsboard.server.edqs.processor.EdqsProducer; +import org.thingsboard.server.edqs.util.VersionsStore; +import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; +import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; +import org.thingsboard.server.queue.common.consumer.QueueStateService; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.edqs.EdqsQueueFactory; +import org.thingsboard.server.queue.edqs.KafkaEdqsComponent; + +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +@Service +@RequiredArgsConstructor +@KafkaEdqsComponent +@Slf4j +public class KafkaEdqsStateService implements EdqsStateService { + + private final EdqsConfig config; + private final EdqsPartitionService partitionService; + private final EdqsQueueFactory queueFactory; + private final TopicService topicService; + @Autowired @Lazy + private EdqsProcessor edqsProcessor; + + private PartitionedQueueConsumerManager> stateConsumer; + private QueueStateService, TbProtoQueueMsg> queueStateService; + private QueueConsumerManager> eventsToBackupConsumer; + private EdqsProducer stateProducer; + + private final VersionsStore versionsStore = new VersionsStore(); + private final AtomicInteger stateReadCount = new AtomicInteger(); + private final AtomicInteger eventsReadCount = new AtomicInteger(); + private Boolean ready; + + @Override + public void init(PartitionedQueueConsumerManager> eventConsumer) { + stateConsumer = PartitionedQueueConsumerManager.>create() + .queueKey(new QueueKey(ServiceType.EDQS, EdqsQueue.STATE.getTopic())) + .topic(EdqsQueue.STATE.getTopic()) + .pollInterval(config.getPollInterval()) + .msgPackProcessor((msgs, consumer, config) -> { + for (TbProtoQueueMsg queueMsg : msgs) { + try { + ToEdqsMsg msg = queueMsg.getValue(); + edqsProcessor.process(msg, EdqsQueue.STATE); + if (stateReadCount.incrementAndGet() % 100000 == 0) { + log.info("[state] Processed {} msgs", stateReadCount.get()); + } + } catch (Exception e) { + log.error("Failed to process message: {}", queueMsg, e); + } + } + consumer.commit(); + }) + .consumerCreator((config, partitionId) -> queueFactory.createEdqsMsgConsumer(EdqsQueue.STATE)) + .consumerExecutor(eventConsumer.getConsumerExecutor()) + .taskExecutor(eventConsumer.getTaskExecutor()) + .scheduler(eventConsumer.getScheduler()) + .uncaughtErrorHandler(edqsProcessor.getErrorHandler()) + .build(); + queueStateService = new QueueStateService<>(); + queueStateService.init(stateConsumer, eventConsumer); + + eventsToBackupConsumer = QueueConsumerManager.>builder() + .name("edqs-events-to-backup-consumer") + .pollInterval(config.getPollInterval()) + .msgPackProcessor((msgs, consumer) -> { + for (TbProtoQueueMsg queueMsg : msgs) { + if (consumer.isStopped()) { + return; + } + try { + ToEdqsMsg msg = queueMsg.getValue(); + log.trace("Processing message: {}", msg); + + if (msg.hasEventMsg()) { + EdqsEventMsg eventMsg = msg.getEventMsg(); + String key = eventMsg.getKey(); + int count = eventsReadCount.incrementAndGet(); + if (count % 100000 == 0) { + log.info("[events-to-backup] Processed {} msgs", count); + } + if (eventMsg.hasVersion()) { + if (!versionsStore.isNew(key, eventMsg.getVersion())) { + continue; + } + } + + TenantId tenantId = getTenantId(msg); + ObjectType objectType = ObjectType.valueOf(eventMsg.getObjectType()); + EdqsEventType eventType = EdqsEventType.valueOf(eventMsg.getEventType()); + log.trace("[{}] Saving to backup [{}] [{}] [{}]", tenantId, objectType, eventType, key); + stateProducer.send(tenantId, objectType, key, msg); + } + } catch (Throwable t) { + log.error("Failed to process message: {}", queueMsg, t); + } + } + consumer.commit(); + }) + .consumerCreator(() -> queueFactory.createEdqsMsgConsumer(EdqsQueue.EVENTS, "events-to-backup-consumer-group")) // shared by all instances consumer group + .consumerExecutor(eventConsumer.getConsumerExecutor()) + .threadPrefix("edqs-events-to-backup") + .build(); + + stateProducer = EdqsProducer.builder() + .queue(EdqsQueue.STATE) + .partitionService(partitionService) + .topicService(topicService) + .producer(queueFactory.createEdqsMsgProducer(EdqsQueue.STATE)) + .build(); + } + + @Override + public void process(Set partitions) { + if (queueStateService.getPartitions() == null) { + eventsToBackupConsumer.subscribe(); + eventsToBackupConsumer.launch(); + } + queueStateService.update(partitions); + } + + @Override + public void save(TenantId tenantId, ObjectType type, String key, EdqsEventType eventType, ToEdqsMsg msg) { + // do nothing here, backup is done by events consumer + } + + @Override + public boolean isReady() { + if (ready == null) { + Set partitionsInProgress = queueStateService.getPartitionsInProgress(); + if (partitionsInProgress != null && partitionsInProgress.isEmpty()) { + ready = true; // once true - always true, not to change readiness status on each repartitioning + } + } + return ready != null && ready; + } + + private TenantId getTenantId(ToEdqsMsg edqsMsg) { + return TenantId.fromUUID(new UUID(edqsMsg.getTenantIdMSB(), edqsMsg.getTenantIdLSB())); + } + + @Override + public void stop() { + stateConsumer.stop(); + stateConsumer.awaitStop(); + eventsToBackupConsumer.stop(); + stateProducer.stop(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java new file mode 100644 index 0000000000..383115ddf1 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java @@ -0,0 +1,98 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.state; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.edqs.processor.EdqsProcessor; +import org.thingsboard.server.edqs.util.EdqsRocksDb; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.edqs.InMemoryEdqsComponent; + +import java.util.Set; + +import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTopic; + +@Service +@RequiredArgsConstructor +@InMemoryEdqsComponent +@Slf4j +public class LocalEdqsStateService implements EdqsStateService { + + private final EdqsRocksDb db; + @Autowired @Lazy + private EdqsProcessor processor; + + private PartitionedQueueConsumerManager> eventConsumer; + private Set partitions; + + @Override + public void init(PartitionedQueueConsumerManager> eventConsumer) { + this.eventConsumer = eventConsumer; + } + + @Override + public void process(Set partitions) { + if (this.partitions == null) { + db.forEach((key, value) -> { + try { + ToEdqsMsg edqsMsg = ToEdqsMsg.parseFrom(value); + log.trace("[{}] Restored msg from RocksDB: {}", key, edqsMsg); + processor.process(edqsMsg, EdqsQueue.STATE); + } catch (Exception e) { + log.error("[{}] Failed to restore value", key, e); + } + }); + log.info("Restore completed"); + } + eventConsumer.update(withTopic(partitions, EdqsQueue.EVENTS.getTopic())); + this.partitions = partitions; + } + + @Override + public void save(TenantId tenantId, ObjectType type, String key, EdqsEventType eventType, ToEdqsMsg msg) { + log.trace("Save to RocksDB: {} {} {} {}", tenantId, type, key, msg); + try { + if (eventType == EdqsEventType.DELETED) { + db.delete(key); + } else { + db.put(key, msg.toByteArray()); + } + } catch (Exception e) { + log.error("[{}] Failed to save event {}", key, msg, e); + } + } + + @Override + public boolean isReady() { + return partitions != null; + } + + @Override + public void stop() { + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java new file mode 100644 index 0000000000..a12a12dbe3 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java @@ -0,0 +1,94 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.stats; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.common.stats.StatsType; +import org.thingsboard.server.queue.edqs.EdqsComponent; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +@EdqsComponent +@Service +@Slf4j +@RequiredArgsConstructor +@ConditionalOnProperty(name = "queue.edqs.stats.enabled", havingValue = "true", matchIfMissing = true) +public class EdqsStatsService { + + private final ConcurrentHashMap statsMap = new ConcurrentHashMap<>(); + private final StatsFactory statsFactory; + + @Scheduled(initialDelayString = "${queue.edqs.stats.print-interval-ms:300000}", + fixedDelayString = "${queue.edqs.stats.print-interval-ms:300000}") + private void reportStats() { + if (statsMap.isEmpty()) { + return; + } + String values = statsMap.entrySet().stream() + .map(kv -> "TenantId [" + kv.getKey() + "] stats [" + kv.getValue() + "]") + .collect(Collectors.joining(System.lineSeparator())); + log.info("EDQS Stats: {}", values); + } + + public void reportEvent(TenantId tenantId, ObjectType objectType, EdqsEventType eventType) { + statsMap.computeIfAbsent(tenantId, id -> new EdqsStats(tenantId, statsFactory)) + .reportEvent(objectType, eventType); + } + + @Getter + @AllArgsConstructor + static class EdqsStats { + + private final TenantId tenantId; + private final ConcurrentHashMap entityCounters = new ConcurrentHashMap<>(); + private final StatsFactory statsFactory; + + private AtomicInteger getOrCreateObjectCounter(ObjectType objectType) { + return entityCounters.computeIfAbsent(objectType, + type -> statsFactory.createGauge(StatsType.EDQS.getName() + "_object_count", new AtomicInteger(), + "tenantId", tenantId.toString(), "objectType", type.name())); + } + + @Override + public String toString() { + return entityCounters.entrySet().stream() + .map(counters -> counters.getKey().name()+ " total = [" + counters.getValue() + "]") + .collect(Collectors.joining(", ")); + } + + public void reportEvent(ObjectType objectType, EdqsEventType eventType) { + AtomicInteger objectCounter = getOrCreateObjectCounter(objectType); + if (eventType == EdqsEventType.UPDATED){ + objectCounter.incrementAndGet(); + } else if (eventType == EdqsEventType.DELETED) { + objectCounter.decrementAndGet(); + } + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java new file mode 100644 index 0000000000..5b4cd7ac4a --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java @@ -0,0 +1,253 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.google.protobuf.ByteString; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.AttributeKv; +import org.thingsboard.server.common.data.edqs.DataPoint; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.Entity; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.edqs.fields.FieldsUtil; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.edqs.data.dp.BoolDataPoint; +import org.thingsboard.server.edqs.data.dp.CompressedJsonDataPoint; +import org.thingsboard.server.edqs.data.dp.CompressedStringDataPoint; +import org.thingsboard.server.edqs.data.dp.DoubleDataPoint; +import org.thingsboard.server.edqs.data.dp.JsonDataPoint; +import org.thingsboard.server.edqs.data.dp.LongDataPoint; +import org.thingsboard.server.edqs.data.dp.StringDataPoint; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.DataPointProto; +import org.xerial.snappy.Snappy; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +@Service +@Slf4j +public class EdqsConverter { + + private final Map> converters = new HashMap<>(); + private final Converter defaultConverter = new JsonConverter<>(Entity.class); + + { + converters.put(ObjectType.RELATION, new JsonConverter<>(EntityRelation.class)); + converters.put(ObjectType.ATTRIBUTE_KV, new Converter() { + @Override + public byte[] serialize(ObjectType type, AttributeKv attributeKv) { + var proto = TransportProtos.AttributeKvProto.newBuilder() + .setEntityIdMSB(attributeKv.getEntityId().getId().getMostSignificantBits()) + .setEntityIdLSB(attributeKv.getEntityId().getId().getLeastSignificantBits()) + .setEntityType(ProtoUtils.toProto(attributeKv.getEntityId().getEntityType())) + .setScope(TransportProtos.AttributeScopeProto.forNumber(attributeKv.getScope().ordinal())) + .setKey(attributeKv.getKey()) + .setVersion(attributeKv.getVersion()); + if (attributeKv.getLastUpdateTs() != null && attributeKv.getValue() != null) { + proto.setDataPoint(toDataPointProto(attributeKv.getLastUpdateTs(), attributeKv.getValue())); + } + return proto.build().toByteArray(); + } + + @Override + public AttributeKv deserialize(ObjectType type, byte[] bytes) throws Exception { + TransportProtos.AttributeKvProto proto = TransportProtos.AttributeKvProto.parseFrom(bytes); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(ProtoUtils.fromProto(proto.getEntityType()), + new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + AttributeScope scope = AttributeScope.values()[proto.getScope().getNumber()]; + DataPoint dataPoint = proto.hasDataPoint() ? fromDataPointProto(proto.getDataPoint()) : null; + return AttributeKv.builder() + .entityId(entityId) + .scope(scope) + .key(proto.getKey()) + .version(proto.getVersion()) + .dataPoint(dataPoint) + .build(); + } + }); + converters.put(ObjectType.LATEST_TS_KV, new Converter() { + @Override + public byte[] serialize(ObjectType type, LatestTsKv latestTsKv) { + var proto = TransportProtos.LatestTsKvProto.newBuilder() + .setEntityIdMSB(latestTsKv.getEntityId().getId().getMostSignificantBits()) + .setEntityIdLSB(latestTsKv.getEntityId().getId().getLeastSignificantBits()) + .setEntityType(ProtoUtils.toProto(latestTsKv.getEntityId().getEntityType())) + .setKey(latestTsKv.getKey()) + .setVersion(latestTsKv.getVersion()); + if (latestTsKv.getTs() != null && latestTsKv.getValue() != null) { + proto.setDataPoint(toDataPointProto(latestTsKv.getTs(), latestTsKv.getValue())); + } + return proto.build().toByteArray(); + } + + @Override + public LatestTsKv deserialize(ObjectType type, byte[] bytes) throws Exception { + TransportProtos.LatestTsKvProto proto = TransportProtos.LatestTsKvProto.parseFrom(bytes); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(ProtoUtils.fromProto(proto.getEntityType()), + new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + DataPoint dataPoint = proto.hasDataPoint() ? fromDataPointProto(proto.getDataPoint()) : null; + return LatestTsKv.builder() + .entityId(entityId) + .key(proto.getKey()) + .version(proto.getVersion()) + .dataPoint(dataPoint) + .build(); + } + }); + } + + public static DataPointProto toDataPointProto(long ts, KvEntry kvEntry) { + DataPointProto.Builder proto = DataPointProto.newBuilder(); + proto.setTs(ts); + switch (kvEntry.getDataType()) { + case BOOLEAN -> proto.setBoolV(kvEntry.getBooleanValue().get()); + case LONG -> proto.setLongV(kvEntry.getLongValue().get()); + case DOUBLE -> proto.setDoubleV(kvEntry.getDoubleValue().get()); + case STRING -> { + String strValue = kvEntry.getStrValue().get(); + if (strValue.length() < CompressedStringDataPoint.MIN_STR_SIZE_TO_COMPRESS) { + proto.setStringV(strValue); + } else { + proto.setCompressedStringV(ByteString.copyFrom(compress(strValue))); + } + } + case JSON -> { + String jsonValue = kvEntry.getJsonValue().get(); + if (jsonValue.length() < CompressedStringDataPoint.MIN_STR_SIZE_TO_COMPRESS) { + proto.setJsonV(jsonValue); + } else { + proto.setCompressedJsonV(ByteString.copyFrom(compress(jsonValue))); + } + } + } + return proto.build(); + } + + public static DataPoint fromDataPointProto(DataPointProto proto) { + long ts = proto.getTs(); + if (proto.hasBoolV()) { + return new BoolDataPoint(ts, proto.getBoolV()); + } else if (proto.hasLongV()) { + return new LongDataPoint(ts, proto.getLongV()); + } else if (proto.hasDoubleV()) { + return new DoubleDataPoint(ts, proto.getDoubleV()); + } else if (proto.hasStringV()) { + return new StringDataPoint(ts, proto.getStringV()); + } else if (proto.hasCompressedStringV()) { + return new CompressedStringDataPoint(ts, proto.getCompressedStringV().toByteArray()); + } else if (proto.hasJsonV()) { + return new JsonDataPoint(ts, proto.getJsonV()); + } else if (proto.hasCompressedJsonV()) { + return new CompressedJsonDataPoint(ts, proto.getCompressedJsonV().toByteArray()); + } else { + throw new IllegalArgumentException("Unsupported data point proto: " + proto); + } + } + + @SneakyThrows + private static byte[] compress(String value) { + byte[] compressed = Snappy.compress(value); + // TODO: limit the size + log.debug("Compressed {} bytes to {} bytes", value.length(), compressed.length); + return compressed; + } + + public static Entity toEntity(EntityType entityType, Object entity) { + Entity edqsEntity = new Entity(); + edqsEntity.setType(entityType); + edqsEntity.setFields(FieldsUtil.toFields(entity)); + return edqsEntity; + } + + public EdqsObject check(ObjectType type, Object object) { + if (object instanceof EdqsObject edqsObject) { + return edqsObject; + } else { + return toEntity(type.toEntityType(), object); + } + } + + @SuppressWarnings("unchecked") + @SneakyThrows + public byte[] serialize(ObjectType type, T value) { + Converter converter = (Converter) converters.get(type); + if (converter != null) { + return converter.serialize(type, value); + } else { + return defaultConverter.serialize(type, (Entity) value); + } + } + + @SneakyThrows + public EdqsObject deserialize(ObjectType type, byte[] bytes) { + Converter converter = converters.get(type); + if (converter != null) { + return converter.deserialize(type, bytes); + } else { + return defaultConverter.deserialize(type, bytes); + } + } + + @RequiredArgsConstructor + private static class JsonConverter implements Converter { + + private static final ObjectMapper mapper = JsonMapper.builder() + .visibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY) + .visibility(PropertyAccessor.GETTER, JsonAutoDetect.Visibility.NONE) + .visibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE) + .build(); + + private final Class type; + + @SneakyThrows + @Override + public byte[] serialize(ObjectType objectType, T value) { + return mapper.writeValueAsBytes(value); + } + + @SneakyThrows + @Override + public T deserialize(ObjectType objectType, byte[] bytes) { + return mapper.readValue(bytes, this.type); + } + + } + + private interface Converter { + + byte[] serialize(ObjectType type, T value) throws Exception; + + T deserialize(ObjectType type, byte[] bytes) throws Exception; + + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java new file mode 100644 index 0000000000..4a991432c7 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.Getter; +import org.rocksdb.Options; +import org.rocksdb.WriteOptions; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.edqs.InMemoryEdqsComponent; + +import java.nio.file.Files; +import java.nio.file.Path; + +@Component +@InMemoryEdqsComponent +public class EdqsRocksDb extends TbRocksDb { + + @Getter + private boolean isNew; + + public EdqsRocksDb(@Value("${queue.edqs.local.rocksdb_path:${user.home}/.rocksdb/edqs}") String path) { + super(path, new Options().setCreateIfMissing(true), new WriteOptions()); + } + + @PostConstruct + @Override + public void init() { + isNew = !Files.exists(Path.of(path)); + super.init(); + } + + @PreDestroy + @Override + public void close() { + super.close(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java new file mode 100644 index 0000000000..970f8585dd --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java @@ -0,0 +1,392 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.DataPoint; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.BooleanFilterPredicate; +import org.thingsboard.server.common.data.query.ComplexFilterPredicate; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.common.data.query.EntityNameFilter; +import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.common.data.query.FilterPredicateType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.KeyFilterPredicate; +import org.thingsboard.server.common.data.query.NumericFilterPredicate; +import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.data.query.SingleEntityFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.query.TsValue; +import org.thingsboard.server.common.data.util.CollectionsUtil; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.DataKey; +import org.thingsboard.server.edqs.query.EdqsCountQuery; +import org.thingsboard.server.edqs.query.EdqsDataQuery; +import org.thingsboard.server.edqs.query.EdqsFilter; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.KeyDictionary; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import static org.apache.commons.lang3.StringUtils.containsIgnoreCase; +import static org.thingsboard.server.common.data.StringUtils.equalsAny; +import static org.thingsboard.server.common.data.StringUtils.splitByCommaWithoutQuotes; +import static org.thingsboard.server.common.data.query.ComplexFilterPredicate.ComplexOperation.AND; +import static org.thingsboard.server.common.data.query.ComplexFilterPredicate.ComplexOperation.OR; + +@Slf4j +public class RepositoryUtils { + + public static final Comparator SORT_ASC = Comparator.comparing((SortableEntityData sed) -> Optional.ofNullable(sed.getSortValue()).orElse("")) + .thenComparing(sp -> sp.getId().toString()); + + public static final Comparator SORT_DESC = Comparator.comparing((SortableEntityData sed) -> Optional.ofNullable(sed.getSortValue()).orElse("")) + .thenComparing(sp -> sp.getId().toString()).reversed(); + + public static EntityType resolveEntityType(EntityFilter entityFilter) { + return switch (entityFilter.getType()) { + case SINGLE_ENTITY -> ((SingleEntityFilter) entityFilter).getSingleEntity().getEntityType(); + case ENTITY_LIST -> ((EntityListFilter) entityFilter).getEntityType(); + case ENTITY_NAME -> ((EntityNameFilter) entityFilter).getEntityType(); + case ENTITY_TYPE -> ((EntityTypeFilter) entityFilter).getEntityType(); + case ASSET_TYPE, ASSET_SEARCH_QUERY -> EntityType.ASSET; + case DEVICE_TYPE, DEVICE_SEARCH_QUERY -> EntityType.DEVICE; + case ENTITY_VIEW_TYPE, ENTITY_VIEW_SEARCH_QUERY -> EntityType.ENTITY_VIEW; + case EDGE_TYPE, EDGE_SEARCH_QUERY -> EntityType.EDGE; + case RELATIONS_QUERY -> { + RelationsQueryFilter rgf = (RelationsQueryFilter) entityFilter; + yield rgf.isMultiRoot() ? rgf.getMultiRootEntitiesType() : rgf.getRootEntity().getEntityType(); + } + case API_USAGE_STATE -> EntityType.API_USAGE_STATE; + }; + } + + public static boolean customerUserIsTryingToAccessTenantEntity(QueryContext ctx, EntityFilter entityFilter) { + if (ctx.isTenantUser()) { + return false; + } else { + return switch (entityFilter.getType()) { + case SINGLE_ENTITY -> { + SingleEntityFilter seFilter = (SingleEntityFilter) entityFilter; + yield isSystemOrTenantEntity(seFilter.getSingleEntity().getEntityType()); + } + case ENTITY_LIST -> { + EntityListFilter elFilter = (EntityListFilter) entityFilter; + yield isSystemOrTenantEntity(elFilter.getEntityType()); + } + case ENTITY_NAME -> { + EntityNameFilter enFilter = (EntityNameFilter) entityFilter; + yield isSystemOrTenantEntity(enFilter.getEntityType()); + } + case ENTITY_TYPE -> { + EntityTypeFilter etFilter = (EntityTypeFilter) entityFilter; + yield isSystemOrTenantEntity(etFilter.getEntityType()); + } + default -> false; + }; + } + } + + private static boolean isSystemOrTenantEntity(EntityType entityType) { + return switch (entityType) { + case DEVICE_PROFILE, ASSET_PROFILE, RULE_CHAIN, TENANT, + TENANT_PROFILE, WIDGET_TYPE, WIDGETS_BUNDLE -> true; + default -> false; + }; + } + + public static EdqsDataQuery toNewQuery(EntityDataQuery oldQuery) { + var query = EdqsDataQuery.builder(); + query.page(oldQuery.getPageLink().getPage()); + query.pageSize(oldQuery.getPageLink().getPageSize()); + query.textSearch(oldQuery.getPageLink().getTextSearch()); + var sortOrder = oldQuery.getPageLink().getSortOrder(); + if (sortOrder != null && toNewKey(sortOrder.getKey()) != null) { + query.sortKey(toNewKey(sortOrder.getKey())); + query.sortDirection(sortOrder.getDirection()); + } else { + query.sortKey(new DataKey(EntityKeyType.ENTITY_FIELD, "createdTime", null)); + query.sortDirection(EntityDataSortOrder.Direction.DESC); + } + query.entityFilter(oldQuery.getEntityFilter()); + query.keyFilters(toKeyFilters(oldQuery.getKeyFilters())); + query.entityFields(toNewKeys(oldQuery.getEntityFields())); + query.latestValues(toNewKeys(oldQuery.getLatestValues())); + return query.build(); + } + + public static EdqsCountQuery toNewQuery(EntityCountQuery oldQuery) { + return EdqsCountQuery.builder() + .entityFilter(oldQuery.getEntityFilter()) + .hasKeyFilters(CollectionsUtil.isNotEmpty(oldQuery.getKeyFilters())) + .keyFilters(toKeyFilters(oldQuery.getKeyFilters())) + .build(); + } + + private static List toKeyFilters(List keyFilters) { + if (keyFilters == null || keyFilters.isEmpty()) { + return Collections.emptyList(); + } else { + List result = new ArrayList<>(); + for (KeyFilter entityFilter : keyFilters) { + var newKey = toNewKey(entityFilter.getKey()); + if (newKey != null) { + result.add(new EdqsFilter(newKey, entityFilter.getValueType(), entityFilter.getPredicate())); + } + } + return result; + } + } + + private static DataKey toNewKey(EntityKey entityKey) { + if (EntityKeyType.ENTITY_FIELD.equals(entityKey.getType())) { + return new DataKey(entityKey.getType(), entityKey.getKey(), null); + } + Integer keyId = KeyDictionary.get(entityKey.getKey()); + if (keyId != null) { + return new DataKey(entityKey.getType(), entityKey.getKey(), keyId); + } else { + log.warn("Missing dictionary key for {}", entityKey.getKey()); + return null; + } + } + + private static List toNewKeys(List entityKeys) { + if (entityKeys == null || entityKeys.isEmpty()) { + return Collections.emptyList(); + } else { + var result = new ArrayList(entityKeys.size()); + for (EntityKey entityKey : entityKeys) { + var newKey = toNewKey(entityKey); + if (newKey != null) { + result.add(newKey); + } + } + return result; + } + } + + public static boolean checkKeyFilters(EntityData entity, List keyFilters) { + for (EdqsFilter keyFilter : keyFilters) { + EntityKeyValueType valueType = keyFilter.valueType(); + if (valueType == null) { + valueType = switch (keyFilter.predicate().getType()) { + case STRING -> EntityKeyValueType.STRING; + case NUMERIC -> EntityKeyValueType.NUMERIC; + case BOOLEAN -> EntityKeyValueType.BOOLEAN; + default -> throw new IllegalStateException(); + }; + } + DataKey dataKey = keyFilter.key(); + DataPoint dp = entity.getDataPoint(dataKey, null); + boolean checkResult = switch (valueType) { + case STRING -> { + String str = dp != null ? dp.valueToString() : null; + yield (dataKey.type() == EntityKeyType.ENTITY_FIELD) ? (str == null || checkKeyFilter(str, keyFilter.predicate())) : + (str != null && checkKeyFilter(str, keyFilter.predicate())); + } + case BOOLEAN -> { + Boolean booleanValue = dp != null ? dp.getBool() : null; + yield booleanValue != null && checkKeyFilter(booleanValue, keyFilter.predicate()); + } + case DATE_TIME, NUMERIC -> { + Double doubleValue = dp != null ? dp.getDouble() : null; + yield doubleValue != null && checkKeyFilter(doubleValue, keyFilter.predicate()); + } + }; + if (!checkResult) { + return false; + } + } + return true; + } + + public static boolean checkKeyFilter(String value, KeyFilterPredicate keyFilterPredicate) { + if (keyFilterPredicate.getType() == FilterPredicateType.COMPLEX) { + return checkComplexKeyFilter(value, (ComplexFilterPredicate) keyFilterPredicate, RepositoryUtils::checkKeyFilter); + } + if (keyFilterPredicate.getType() != FilterPredicateType.STRING) { + throw new IllegalStateException("Not implemented"); + } + StringFilterPredicate predicate = (StringFilterPredicate) keyFilterPredicate; + String predicateValue = predicate.getValue().getValue(); + if (StringUtils.isEmpty(predicateValue)) { + return true; + } + if (predicate.isIgnoreCase()) { + predicateValue = predicateValue.toLowerCase(); + value = value.toLowerCase(); + } + return switch (predicate.getOperation()) { + case EQUAL -> value.equals(predicateValue); + case STARTS_WITH -> value.startsWith(predicateValue); + case ENDS_WITH -> value.endsWith(predicateValue); + case NOT_EQUAL -> !value.equals(predicateValue); + case CONTAINS -> value.contains(predicateValue); + case NOT_CONTAINS -> !value.contains(predicateValue); + case IN -> equalsAny(value, splitByCommaWithoutQuotes(predicateValue)); + case NOT_IN -> !equalsAny(value, splitByCommaWithoutQuotes(predicateValue)); + }; + } + + public static boolean checkKeyFilter(Double value, KeyFilterPredicate keyFilterPredicate) { + if (keyFilterPredicate.getType() == FilterPredicateType.COMPLEX) { + return checkComplexKeyFilter(value, (ComplexFilterPredicate) keyFilterPredicate, RepositoryUtils::checkKeyFilter); + } + if (keyFilterPredicate.getType() != FilterPredicateType.NUMERIC) { + throw new IllegalStateException("Not implemented"); + } + NumericFilterPredicate predicate = (NumericFilterPredicate) keyFilterPredicate; + Double predicateValue = predicate.getValue().getValue(); + return switch (predicate.getOperation()) { + case EQUAL -> value.equals(predicateValue); + case NOT_EQUAL -> !value.equals(predicateValue); + case GREATER -> value.compareTo(predicateValue) > 0; + case LESS -> value.compareTo(predicateValue) < 0; + case GREATER_OR_EQUAL -> value.compareTo(predicateValue) >= 0; + case LESS_OR_EQUAL -> value.compareTo(predicateValue) <= 0; + }; + } + + public static boolean checkKeyFilter(Boolean value, KeyFilterPredicate keyFilterPredicate) { + if (keyFilterPredicate.getType() == FilterPredicateType.COMPLEX) { + return checkComplexKeyFilter(value, (ComplexFilterPredicate) keyFilterPredicate, RepositoryUtils::checkKeyFilter); + } + if (keyFilterPredicate.getType() != FilterPredicateType.BOOLEAN) { + throw new IllegalStateException("Not implemented"); + } + BooleanFilterPredicate predicate = (BooleanFilterPredicate) keyFilterPredicate; + Boolean predicateValue = predicate.getValue().getValue(); + return switch (predicate.getOperation()) { + case EQUAL -> value.equals(predicateValue); + case NOT_EQUAL -> !value.equals(predicateValue); + }; + } + + public static boolean checkComplexKeyFilter(T value, ComplexFilterPredicate filterPredicates, + SimpleKeyFilter simpleKeyFilter) { + if (filterPredicates.getOperation() == AND) { + for (KeyFilterPredicate filterPredicate : filterPredicates.getPredicates()) { + if (!simpleKeyFilter.check(value, filterPredicate)) { + return false; + } + } + return true; + } else if (filterPredicates.getOperation() == OR) { + for (KeyFilterPredicate filterPredicate : filterPredicates.getPredicates()) { + if (simpleKeyFilter.check(value, filterPredicate)) { + return true; + } + } + return false; + } else { + return false; + } + } + + public static Pattern toSqlLikePattern(String nameFilter) { + if (StringUtils.isNotBlank(nameFilter)) { + boolean percentSymbolOnStart = nameFilter.startsWith("%"); + boolean percentSymbolOnEnd = nameFilter.endsWith("%"); + if (percentSymbolOnStart) { + nameFilter = nameFilter.substring(1); + } + if (percentSymbolOnEnd) { + nameFilter = nameFilter.substring(0, nameFilter.length() - 1); + } + if (percentSymbolOnStart || percentSymbolOnEnd) { + return Pattern.compile((percentSymbolOnStart ? ".*" : "") + Pattern.quote(nameFilter) + (percentSymbolOnEnd ? ".*" : ""), Pattern.CASE_INSENSITIVE); + } else { + return Pattern.compile(Pattern.quote(nameFilter) + ".*", Pattern.CASE_INSENSITIVE); + } + } + return null; + } + + @FunctionalInterface + public interface SimpleKeyFilter { + + boolean check(T value, KeyFilterPredicate predicate); + + } + + public static TsValue toTsValue(long ts, DataPoint dp) { + if (dp != null) { + return new TsValue(dp.getTs() > 0 ? dp.getTs() : ts, dp.valueToString()); + } else { + return new TsValue(ts, ""); + } + } + + public static String getSortValue(EntityData entity, DataKey sortKey) { + if (sortKey == null) { + return null; + } + switch (sortKey.type()) { + case ENTITY_FIELD -> { + return entity.getField(sortKey.key()); + } + case ATTRIBUTE, CLIENT_ATTRIBUTE, SHARED_ATTRIBUTE, SERVER_ATTRIBUTE -> { + var dp = entity.getAttr(sortKey.keyId(), sortKey.type()); + return dp != null ? dp.valueToString() : ""; + } + case TIME_SERIES -> { + var dp = entity.getTs(sortKey.keyId()); + return dp != null ? dp.valueToString() : ""; + } + default -> throw new IllegalStateException("toSortKey is not implemented for type: " + sortKey.type()); + } + } + + public static boolean checkFilters(EdqsQuery query, EntityData entity) { + if (entity == null || entity.getFields() == null) { + return false; // Entity was already removed or not arrived yet; + } + if (query.isHasKeyFilters() && !checkKeyFilters(entity, query.getKeyFilters())) { + return false; + } + if (query instanceof EdqsDataQuery dataQuery) { + return !dataQuery.isHasTextSearch() || checkTextSearch(entity, dataQuery); + } + return true; + } + + private static boolean checkTextSearch(EntityData entityData, EdqsDataQuery query) { + return Stream.concat(query.getEntityFields().stream(), query.getLatestValues().stream()) + .anyMatch(key -> { + DataPoint value = entityData.getDataPoint(key, null); + return value != null && containsIgnoreCase(value.valueToString(), query.getTextSearch()); + }); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbBytePool.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbBytePool.java new file mode 100644 index 0000000000..3b135be59c --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbBytePool.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import com.google.common.hash.Hashing; +import org.springframework.util.ConcurrentReferenceHashMap; + +import java.util.concurrent.ConcurrentMap; + +public class TbBytePool { + + private static final ConcurrentMap pool = new ConcurrentReferenceHashMap<>(); + + public static byte[] intern(byte[] data) { + if (data == null) { + return null; + } + var checksum = Hashing.sha512().hashBytes(data).toString(); + return pool.computeIfAbsent(checksum, c -> data); + } + + public static int size(){ + return pool.size(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbRocksDb.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbRocksDb.java new file mode 100644 index 0000000000..23f2fa2c9e --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbRocksDb.java @@ -0,0 +1,77 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import lombok.SneakyThrows; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksIterator; +import org.rocksdb.WriteOptions; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.function.BiConsumer; + +public class TbRocksDb { + + protected final String path; + private final Options dbOptions; + private final WriteOptions writeOptions; + protected RocksDB db; + + static { + RocksDB.loadLibrary(); + } + + public TbRocksDb(String path, Options dbOptions, WriteOptions writeOptions) { + this.path = path; + this.dbOptions = dbOptions; + this.writeOptions = writeOptions; + } + + @SneakyThrows + public void init() { + Files.createDirectories(Path.of(path).getParent()); + db = RocksDB.open(dbOptions, path); + } + + @SneakyThrows + public void put(String key, byte[] value) { + db.put(writeOptions, key.getBytes(StandardCharsets.UTF_8), value); + } + + public void forEach(BiConsumer processor) { + try (RocksIterator iterator = db.newIterator()) { + for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { + String key = new String(iterator.key(), StandardCharsets.UTF_8); + processor.accept(key, iterator.value()); + } + } + } + + @SneakyThrows + public void delete(String key) { + db.delete(writeOptions, key.getBytes(StandardCharsets.UTF_8)); + } + + public void close() { + if (db != null) { + db.close(); + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbStringPool.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbStringPool.java new file mode 100644 index 0000000000..9c9c3b5b13 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbStringPool.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import org.springframework.util.ConcurrentReferenceHashMap; + +import java.util.concurrent.ConcurrentMap; + +public class TbStringPool { + + private static final ConcurrentMap pool = new ConcurrentReferenceHashMap<>(); + + public static String intern(String data) { + if (data == null) { + return null; + } + return pool.computeIfAbsent(data, str -> str); + } + + public static int size(){ + return pool.size(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java new file mode 100644 index 0000000000..e52b1bbac9 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import lombok.extern.slf4j.Slf4j; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +@Slf4j +public class VersionsStore { + + private final Cache versions = Caffeine.newBuilder() + .expireAfterWrite(1, TimeUnit.HOURS) + .build(); + + public boolean isNew(String key, Long version) { + AtomicBoolean isNew = new AtomicBoolean(false); + versions.asMap().compute(key, (k, prevVersion) -> { + if (prevVersion == null || prevVersion < version) { + isNew.set(true); + return version; + } else { + if (version < prevVersion) { + log.info("[{}] Version {} is outdated, the latest is {}", key, version, prevVersion); + } + return prevVersion; + } + }); + return isNew.get(); + } + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index d5a6ff99b2..178caf7961 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -133,7 +133,22 @@ public enum MsgType { * Messages that are sent to and from edge session to start edge synchronization process */ EDGE_SYNC_REQUEST_TO_EDGE_SESSION_MSG, - EDGE_SYNC_RESPONSE_FROM_EDGE_SESSION_MSG; + EDGE_SYNC_RESPONSE_FROM_EDGE_SESSION_MSG, + + + CF_INIT_MSG, // Sent to init particular calculated field; + CF_LINK_INIT_MSG, // Sent to init particular calculated field; + CF_STATE_RESTORE_MSG, // Sent to restore particular calculated field entity state; + CF_PARTITIONS_CHANGE_MSG, // Sent when cluster event occures; + + CF_ENTITY_LIFECYCLE_MSG, // Sent on CF/Device/Asset create/update/delete; + CF_TELEMETRY_MSG, // Sent from queue to actor system; + CF_LINKED_TELEMETRY_MSG, // Sent from queue to actor system; + + /* CF Manager Actor -> CF Entity actor */ + CF_ENTITY_TELEMETRY_MSG, + CF_ENTITY_INIT_CF_MSG, + CF_ENTITY_DELETE_MSG; @Getter private final boolean ignoreOnStart; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 5592244bd9..4e0f583285 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -24,6 +24,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -34,8 +35,10 @@ import org.thingsboard.server.common.msg.gen.MsgProtos; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.io.Serializable; +import java.util.List; import java.util.Objects; import java.util.UUID; +import java.util.concurrent.CopyOnWriteArrayList; /** * Created by ashvayka on 13.01.18. @@ -64,6 +67,8 @@ public final class TbMsg implements Serializable { private final UUID correlationId; private final Integer partition; + private final List previousCalculatedFieldIds; + @Getter(value = AccessLevel.NONE) @JsonIgnore //This field is not serialized because we use queues and there is no need to do it @@ -112,7 +117,7 @@ public final class TbMsg implements Serializable { } private TbMsg(String queueName, UUID id, long ts, TbMsgType internalType, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, UUID correlationId, Integer partition, TbMsgProcessingCtx ctx, TbMsgCallback callback) { + RuleChainId ruleChainId, RuleNodeId ruleNodeId, UUID correlationId, Integer partition, List previousCalculatedFieldIds, TbMsgProcessingCtx ctx, TbMsgCallback callback) { this.id = id != null ? id : UUID.randomUUID(); this.queueName = queueName; if (ts > 0) { @@ -139,6 +144,9 @@ public final class TbMsg implements Serializable { this.ruleNodeId = ruleNodeId; this.correlationId = correlationId; this.partition = partition; + this.previousCalculatedFieldIds = previousCalculatedFieldIds != null + ? new CopyOnWriteArrayList<>(previousCalculatedFieldIds) + : new CopyOnWriteArrayList<>(); this.ctx = ctx != null ? ctx : new TbMsgProcessingCtx(); this.callback = Objects.requireNonNullElse(callback, TbMsgCallback.EMPTY); } @@ -186,6 +194,16 @@ public final class TbMsg implements Serializable { builder.setPartition(msg.getPartition()); } + if (msg.getPreviousCalculatedFieldIds() != null) { + for (CalculatedFieldId calculatedFieldId : msg.getPreviousCalculatedFieldIds()) { + MsgProtos.CalculatedFieldIdProto calculatedFieldIdProto = MsgProtos.CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) + .build(); + builder.addCalculatedFields(calculatedFieldIdProto); + } + } + builder.setCtx(msg.ctx.toProto()); return builder.build().toByteArray(); } @@ -200,6 +218,7 @@ public final class TbMsg implements Serializable { RuleNodeId ruleNodeId = null; UUID correlationId = null; Integer partition = null; + List calculatedFieldIds = new CopyOnWriteArrayList<>(); if (proto.getCustomerIdMSB() != 0L && proto.getCustomerIdLSB() != 0L) { customerId = new CustomerId(new UUID(proto.getCustomerIdMSB(), proto.getCustomerIdLSB())); } @@ -214,6 +233,14 @@ public final class TbMsg implements Serializable { partition = proto.getPartition(); } + for (MsgProtos.CalculatedFieldIdProto cfIdProto : proto.getCalculatedFieldsList()) { + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID( + cfIdProto.getCalculatedFieldIdMSB(), + cfIdProto.getCalculatedFieldIdLSB() + )); + calculatedFieldIds.add(calculatedFieldId); + } + TbMsgProcessingCtx ctx; if (proto.hasCtx()) { ctx = TbMsgProcessingCtx.fromProto(proto.getCtx()); @@ -224,7 +251,7 @@ public final class TbMsg implements Serializable { TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), null, proto.getType(), entityId, customerId, - metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, correlationId, partition, ctx, callback); + metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, correlationId, partition, calculatedFieldIds, ctx, callback); } catch (InvalidProtocolBufferException e) { throw new IllegalStateException("Could not parse protobuf for TbMsg", e); } @@ -249,6 +276,7 @@ public final class TbMsg implements Serializable { /** * Checks if the message is still valid for processing. May be invalid if the message pack is timed-out or canceled. + * * @return 'true' if message is valid for processing, 'false' otherwise. */ public boolean isValid() { @@ -343,6 +371,7 @@ public final class TbMsg implements Serializable { protected RuleNodeId ruleNodeId; protected UUID correlationId; protected Integer partition; + protected List previousCalculatedFieldIds; protected TbMsgProcessingCtx ctx; protected TbMsgCallback callback; @@ -363,6 +392,7 @@ public final class TbMsg implements Serializable { this.ruleNodeId = tbMsg.ruleNodeId; this.correlationId = tbMsg.correlationId; this.partition = tbMsg.partition; + this.previousCalculatedFieldIds = tbMsg.previousCalculatedFieldIds; this.ctx = tbMsg.ctx; this.callback = tbMsg.callback; } @@ -385,8 +415,7 @@ public final class TbMsg implements Serializable { /** *

Deprecated: This should only be used when you need to specify a custom message type that doesn't exist in the {@link TbMsgType} enum. * Prefer using {@link #type(TbMsgType)} instead. - * - * */ + */ @Deprecated public TbMsgBuilder type(String type) { this.type = type; @@ -454,6 +483,11 @@ public final class TbMsg implements Serializable { return this; } + public TbMsgBuilder previousCalculatedFieldIds(List previousCalculatedFieldIds) { + this.previousCalculatedFieldIds = new CopyOnWriteArrayList<>(previousCalculatedFieldIds); + return this; + } + public TbMsgBuilder ctx(TbMsgProcessingCtx ctx) { this.ctx = ctx; return this; @@ -465,7 +499,7 @@ public final class TbMsg implements Serializable { } public TbMsg build() { - return new TbMsg(queueName, id, ts, internalType, type, originator, customerId, metaData, dataType, data, ruleChainId, ruleNodeId, correlationId, partition, ctx, callback); + return new TbMsg(queueName, id, ts, internalType, type, originator, customerId, metaData, dataType, data, ruleChainId, ruleNodeId, correlationId, partition, previousCalculatedFieldIds, ctx, callback); } public String toString() { @@ -473,8 +507,8 @@ public final class TbMsg implements Serializable { ", type=" + this.type + ", internalType=" + this.internalType + ", originator=" + this.originator + ", customerId=" + this.customerId + ", metaData=" + this.metaData + ", dataType=" + this.dataType + ", data=" + this.data + ", ruleChainId=" + this.ruleChainId + ", ruleNodeId=" + this.ruleNodeId + - ", correlationId=" + this.correlationId + ", partition=" + this.partition + ", ctx=" + this.ctx + - ", callback=" + this.callback + ")"; + ", correlationId=" + this.correlationId + ", partition=" + this.partition + ", previousCalculatedFields=" + this.previousCalculatedFieldIds + + ", ctx=" + this.ctx + ", callback=" + this.callback + ")"; } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/ToCalculatedFieldSystemMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/ToCalculatedFieldSystemMsg.java new file mode 100644 index 0000000000..c05c0f121e --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/ToCalculatedFieldSystemMsg.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg; + +import org.thingsboard.server.common.msg.aware.TenantAwareMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; + +public interface ToCalculatedFieldSystemMsg extends TenantAwareMsg { + + default TbCallback getCallback() { + return TbCallback.EMPTY; + } + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldEntityLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldEntityLifecycleMsg.java new file mode 100644 index 0000000000..099240f54d --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldEntityLifecycleMsg.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; + +@Data +public class CalculatedFieldEntityLifecycleMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final ComponentLifecycleMsg data; + private final TbCallback callback; + + @Override + public MsgType getMsgType() { + return MsgType.CF_ENTITY_LIFECYCLE_MSG; + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitMsg.java new file mode 100644 index 0000000000..e453d2963c --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitMsg.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; + +@Data +public class CalculatedFieldInitMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final CalculatedField cf; + + @Override + public MsgType getMsgType() { + return MsgType.CF_INIT_MSG; + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldLinkInitMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldLinkInitMsg.java new file mode 100644 index 0000000000..d142eb78d8 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldLinkInitMsg.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; + +@Data +public class CalculatedFieldLinkInitMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final CalculatedFieldLink link; + + @Override + public MsgType getMsgType() { + return MsgType.CF_LINK_INIT_MSG; + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldPartitionChangeMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldPartitionChangeMsg.java new file mode 100644 index 0000000000..38a4853219 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldPartitionChangeMsg.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; + +import java.util.Set; + +@Data +public class CalculatedFieldPartitionChangeMsg implements ToCalculatedFieldSystemMsg { + + private final boolean[] partitions; + + @Override + public TenantId getTenantId() { + return TenantId.SYS_TENANT_ID; + } + + @Override + public MsgType getMsgType() { + return MsgType.CF_PARTITIONS_CHANGE_MSG; + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsApiService.java b/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsApiService.java new file mode 100644 index 0000000000..05864fe863 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsApiService.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.edqs; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.edqs.query.EdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsResponse; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; + +public interface EdqsApiService { + + ListenableFuture processRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request); + + boolean isEnabled(); + + void setEnabled(boolean enabled); + + boolean isSupported(); + + boolean isAutoEnable(); + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsService.java b/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsService.java new file mode 100644 index 0000000000..32ff57e3e0 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsService.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.edqs; + +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +public interface EdqsService { + + void onUpdate(TenantId tenantId, EntityId entityId, Object entity); + + void onUpdate(TenantId tenantId, ObjectType objectType, EdqsObject object); + + void onDelete(TenantId tenantId, EntityId entityId); + + void onDelete(TenantId tenantId, ObjectType objectType, EdqsObject object); + + void processSystemRequest(ToCoreEdqsRequest request); + + void processSystemMsg(ToCoreEdqsMsg request); + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index b429d503d9..13fd6159fc 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.msg.plugin; +import lombok.Builder; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; @@ -25,6 +26,7 @@ import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.common.msg.cluster.ToAllNodesMsg; +import java.io.Serial; import java.util.Optional; /** @@ -32,11 +34,32 @@ import java.util.Optional; */ @Data public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { + + @Serial private static final long serialVersionUID = -5303421482781273062L; private final TenantId tenantId; private final EntityId entityId; private final ComponentLifecycleEvent event; + private final String oldName; + private final String name; + private final EntityId oldProfileId; + private final EntityId profileId; + + public ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event) { + this(tenantId, entityId, event, null, null, null, null); + } + + @Builder + private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId) { + this.tenantId = tenantId; + this.entityId = entityId; + this.event = event; + this.oldName = oldName; + this.name = name; + this.oldProfileId = oldProfileId; + this.profileId = profileId; + } public Optional getRuleChainId() { return entityId.getEntityType() == EntityType.RULE_CHAIN ? Optional.of((RuleChainId) entityId) : Optional.empty(); @@ -46,4 +69,5 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { public MsgType getMsgType() { return MsgType.COMPONENT_LIFE_CYCLE_MSG; } + } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java index 3547ea9120..f31fdfa7a8 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java @@ -26,7 +26,8 @@ public enum ServiceType { TB_RULE_ENGINE("TB Rule Engine"), TB_TRANSPORT("TB Transport"), JS_EXECUTOR("JS Executor"), - TB_VC_EXECUTOR("TB VC Executor"); + TB_VC_EXECUTOR("TB VC Executor"), + EDQS("TB Entity Data Query Service"); private final String label; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java index c8eed097bf..ee8990d931 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java @@ -15,6 +15,10 @@ */ package org.thingsboard.server.common.msg.queue; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.UUID; + public interface TbCallback { TbCallback EMPTY = new TbCallback() { @@ -30,6 +34,10 @@ public interface TbCallback { } }; + default UUID getId(){ + return EntityId.NULL_UUID; + } + void onSuccess(); void onFailure(Throwable t); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java index ddfbd36a33..f09826e9b6 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java @@ -17,41 +17,48 @@ package org.thingsboard.server.common.msg.queue; import lombok.Builder; import lombok.Getter; -import lombok.ToString; import org.thingsboard.server.common.data.id.TenantId; import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; -@ToString public class TopicPartitionInfo { private final String topic; private final TenantId tenantId; private final Integer partition; @Getter + private final boolean useInternalPartition; + @Getter private final String fullTopicName; @Getter private final boolean myPartition; @Builder - public TopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean myPartition) { + public TopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean useInternalPartition, boolean myPartition) { this.topic = topic; this.tenantId = tenantId; this.partition = partition; + this.useInternalPartition = useInternalPartition; this.myPartition = myPartition; String tmp = topic; if (tenantId != null && !tenantId.isNullUid()) { tmp += ".isolated." + tenantId.getId().toString(); } - if (partition != null) { + if (partition != null && !useInternalPartition) { tmp += "." + partition; } this.fullTopicName = tmp; } + public TopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean myPartition) { + this(topic, tenantId, partition, false, myPartition); + } + public TopicPartitionInfo newByTopic(String topic) { - return new TopicPartitionInfo(topic, this.tenantId, this.partition, this.myPartition); + return new TopicPartitionInfo(topic, this.tenantId, this.partition, this.useInternalPartition, this.myPartition); } public String getTopic() { @@ -66,6 +73,18 @@ public class TopicPartitionInfo { return Optional.ofNullable(partition); } + public TopicPartitionInfo withTopic(String topic) { + return new TopicPartitionInfo(topic, this.tenantId, this.partition, this.useInternalPartition, this.myPartition); + } + + public static Set withTopic(Set partitions, String topic) { + return partitions.stream().map(tpi -> tpi.withTopic(topic)).collect(Collectors.toSet()); + } + + public TopicPartitionInfo withUseInternalPartition(boolean useInternalPartition) { + return new TopicPartitionInfo(this.topic, this.tenantId, this.partition, useInternalPartition, this.myPartition); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -79,6 +98,16 @@ public class TopicPartitionInfo { @Override public int hashCode() { - return Objects.hash(fullTopicName); + return Objects.hash(fullTopicName, partition); + } + + @Override + public String toString() { + String str = fullTopicName; + if (useInternalPartition) { + str += "[" + partition + "]"; + } + return str; } + } diff --git a/common/message/src/main/proto/tbmsg.proto b/common/message/src/main/proto/tbmsg.proto index 86bdef30e7..65a967e9e4 100644 --- a/common/message/src/main/proto/tbmsg.proto +++ b/common/message/src/main/proto/tbmsg.proto @@ -70,4 +70,11 @@ message TbMsgProto { int64 correlationIdMSB = 20; int64 correlationIdLSB = 21; int32 partition = 22; + + repeated CalculatedFieldIdProto calculatedFields = 23; +} + +message CalculatedFieldIdProto { + int64 calculatedFieldIdMSB = 1; + int64 calculatedFieldIdLSB = 2; } diff --git a/common/pom.xml b/common/pom.xml index c275227018..9a7a836ba5 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -49,6 +49,7 @@ edge-api version-control script + edqs diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index be210ed4ed..f1da00d943 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ResourceSubType; import org.thingsboard.server.common.data.ResourceType; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; @@ -58,12 +59,15 @@ import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.rpc.RpcError; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; @@ -88,6 +92,7 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceDeleteMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; import java.util.ArrayList; import java.util.Arrays; @@ -111,14 +116,28 @@ public class ProtoUtils { } public static TransportProtos.ComponentLifecycleMsgProto toProto(ComponentLifecycleMsg msg) { - return TransportProtos.ComponentLifecycleMsgProto.newBuilder() + var builder = TransportProtos.ComponentLifecycleMsgProto.newBuilder() .setTenantIdMSB(msg.getTenantId().getId().getMostSignificantBits()) .setTenantIdLSB(msg.getTenantId().getId().getLeastSignificantBits()) .setEntityType(toProto(msg.getEntityId().getEntityType())) .setEntityIdMSB(msg.getEntityId().getId().getMostSignificantBits()) .setEntityIdLSB(msg.getEntityId().getId().getLeastSignificantBits()) - .setEvent(TransportProtos.ComponentLifecycleEvent.forNumber(msg.getEvent().ordinal())) - .build(); + .setEvent(TransportProtos.ComponentLifecycleEvent.forNumber(msg.getEvent().ordinal())); + if (msg.getProfileId() != null) { + builder.setProfileIdMSB(msg.getProfileId().getId().getMostSignificantBits()); + builder.setProfileIdLSB(msg.getProfileId().getId().getLeastSignificantBits()); + } + if (msg.getOldProfileId() != null) { + builder.setOldProfileIdMSB(msg.getOldProfileId().getId().getMostSignificantBits()); + builder.setOldProfileIdLSB(msg.getOldProfileId().getId().getLeastSignificantBits()); + } + if (msg.getName() != null) { + builder.setName(msg.getName()); + } + if (msg.getOldName() != null) { + builder.setOldName(msg.getOldName()); + } + return builder.build(); } public static TransportProtos.EntityTypeProto toProto(EntityType entityType) { @@ -126,18 +145,32 @@ public class ProtoUtils { } public static ComponentLifecycleMsg fromProto(TransportProtos.ComponentLifecycleMsgProto proto) { - return new ComponentLifecycleMsg( - TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), - EntityIdFactory.getByTypeAndUuid(fromProto(proto.getEntityType()), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())), - ComponentLifecycleEvent.values()[proto.getEventValue()] - ); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(fromProto(proto.getEntityType()), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + var builder = ComponentLifecycleMsg.builder() + .tenantId(TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB()))) + .entityId(entityId) + .event(ComponentLifecycleEvent.values()[proto.getEventValue()]); + if (!StringUtils.isEmpty(proto.getName())) { + builder.name(proto.getName()); + } + if (!StringUtils.isEmpty(proto.getOldName())) { + builder.oldName(proto.getOldName()); + } + if (proto.getProfileIdMSB() != 0 || proto.getProfileIdLSB() != 0) { + var profileType = EntityType.DEVICE.equals(entityId.getEntityType()) ? EntityType.DEVICE_PROFILE : EntityType.ASSET_PROFILE; + builder.profileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB()))); + } + if (proto.getOldProfileIdMSB() != 0 || proto.getOldProfileIdLSB() != 0) { + var profileType = EntityType.DEVICE.equals(entityId.getEntityType()) ? EntityType.DEVICE_PROFILE : EntityType.ASSET_PROFILE; + builder.oldProfileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB()))); + } + return builder.build(); } public static EntityType fromProto(TransportProtos.EntityTypeProto entityType) { return entityTypeByProtoNumber[entityType.getNumber()]; } - public static TransportProtos.ToEdgeSyncRequestMsgProto toProto(ToEdgeSyncRequest request) { return TransportProtos.ToEdgeSyncRequestMsgProto.newBuilder() .setTenantIdMSB(request.getTenantId().getId().getMostSignificantBits()) @@ -627,6 +660,96 @@ public class ProtoUtils { return new BaseAttributeKvEntry(entry, proto.getLastUpdateTs(), proto.hasVersion() ? proto.getVersion() : null); } + public static BasicKvEntry basicKvEntryFromProto(TransportProtos.AttributeValueProto proto) { + boolean hasValue = proto.getHasV(); + String key = proto.getKey(); + return switch (proto.getType()) { + case BOOLEAN_V -> new BooleanDataEntry(key, hasValue ? proto.getBoolV() : null); + case LONG_V -> new LongDataEntry(key, hasValue ? proto.getLongV() : null); + case DOUBLE_V -> new DoubleDataEntry(key, hasValue ? proto.getDoubleV() : null); + case STRING_V -> new StringDataEntry(key, hasValue ? proto.getStringV() : null); + case JSON_V -> new JsonDataEntry(key, hasValue ? proto.getJsonV() : null); + default -> null; + }; + } + + public static BasicKvEntry fromProto(KeyValueProto proto) { + String key = proto.getKey(); + return switch (proto.getType()) { + case BOOLEAN_V -> new BooleanDataEntry(key, proto.getBoolV()); + case LONG_V -> new LongDataEntry(key, proto.getLongV()); + case DOUBLE_V -> new DoubleDataEntry(key, proto.getDoubleV()); + case STRING_V -> new StringDataEntry(key, proto.getStringV()); + case JSON_V -> new JsonDataEntry(key, proto.getJsonV()); + default -> null; + }; + } + + public static BasicKvEntry basicKvEntryFromKvEntry(KvEntry kvEntry) { + String key = kvEntry.getKey(); + return switch (kvEntry.getDataType()) { + case BOOLEAN -> new BooleanDataEntry(key, kvEntry.getBooleanValue().orElse(null)); + case LONG -> new LongDataEntry(key, kvEntry.getLongValue().orElse(null)); + case DOUBLE -> new DoubleDataEntry(key, kvEntry.getDoubleValue().orElse(null)); + case STRING -> new StringDataEntry(key, kvEntry.getStrValue().orElse(null)); + case JSON -> new JsonDataEntry(key, kvEntry.getJsonValue().orElse(null)); + }; + } + + public static TsKvEntry fromProto(TransportProtos.TsKvProto proto) { + TransportProtos.KeyValueProto kvProto = proto.getKv(); + String key = kvProto.getKey(); + KvEntry entry = switch (kvProto.getType()) { + case BOOLEAN_V -> new BooleanDataEntry(key, kvProto.getBoolV()); + case LONG_V -> new LongDataEntry(key, kvProto.getLongV()); + case DOUBLE_V -> new DoubleDataEntry(key, kvProto.getDoubleV()); + case STRING_V -> new StringDataEntry(key, kvProto.getStringV()); + case JSON_V -> new JsonDataEntry(key, kvProto.getJsonV()); + default -> null; + }; + return new BasicTsKvEntry(proto.getTs(), entry, proto.hasVersion() ? proto.getVersion() : null); + } + + public static TransportProtos.TsKvProto toTsKvProto(TsKvEntry tsKvEntry) { + var builder = TransportProtos.TsKvProto.newBuilder() + .setTs(tsKvEntry.getTs()) + .setKv(toKeyValueProto(tsKvEntry)); + if (tsKvEntry.getVersion() != null) { + builder.setVersion(tsKvEntry.getVersion()); + } + return builder.build(); + } + + public static TransportProtos.KeyValueProto toKeyValueProto(KvEntry kvEntry) { + TransportProtos.KeyValueProto.Builder builder = TransportProtos.KeyValueProto.newBuilder(); + builder.setKey(kvEntry.getKey()); + switch (kvEntry.getDataType()) { + case BOOLEAN: + builder.setType(TransportProtos.KeyValueType.BOOLEAN_V) + .setBoolV(kvEntry.getBooleanValue().orElse(false)); + break; + case LONG: + builder.setType(TransportProtos.KeyValueType.LONG_V) + .setLongV(kvEntry.getLongValue().orElse(0L)); + break; + case DOUBLE: + builder.setType(TransportProtos.KeyValueType.DOUBLE_V) + .setDoubleV(kvEntry.getDoubleValue().orElse(0.0)); + break; + case STRING: + builder.setType(TransportProtos.KeyValueType.STRING_V) + .setStringV(kvEntry.getStrValue().orElse("")); + break; + case JSON: + builder.setType(TransportProtos.KeyValueType.JSON_V) + .setJsonV(kvEntry.getJsonValue().orElse("{}")); + break; + default: + throw new IllegalArgumentException("Unsupported KvEntry data type: " + kvEntry.getDataType()); + } + return builder.build(); + } + public static TransportProtos.DeviceProto toProto(Device device) { var builder = TransportProtos.DeviceProto.newBuilder() .setTenantIdMSB(device.getTenantId().getId().getMostSignificantBits()) diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 1ecd4e6a3a..6ecdf13981 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -59,6 +59,8 @@ enum EntityTypeProto { DOMAIN = 36; MOBILE_APP = 37; MOBILE_APP_BUNDLE = 38; + CALCULATED_FIELD = 39; + CALCULATED_FIELD_LINK = 40; } /** @@ -70,6 +72,7 @@ message ServiceInfo { repeated string transports = 6; SystemInfoProto systemInfo = 10; repeated string assignedTenantProfiles = 11; + string label = 12; } message SystemInfoProto { @@ -170,17 +173,47 @@ message AttributeValueProto { optional int64 version = 10; } +message AttributeKvProto { + int64 entityIdMSB = 1; + int64 entityIdLSB = 2; + EntityTypeProto entityType = 3; + AttributeScopeProto scope = 4; + string key = 5; + int64 version = 6; + DataPointProto dataPoint = 7; +} + message TsKvProto { int64 ts = 1; KeyValueProto kv = 2; optional int64 version = 3; } +message LatestTsKvProto { + int64 entityIdMSB = 1; + int64 entityIdLSB = 2; + EntityTypeProto entityType = 3; + string key = 4; + int64 version = 5; + DataPointProto dataPoint = 6; +} + message TsKvListProto { int64 ts = 1; repeated KeyValueProto kv = 2; } +message DataPointProto { + int64 ts = 1; + optional bool boolV = 2; + optional int64 longV = 3; + optional double doubleV = 4; + optional string stringV = 5; + optional bytes compressedStringV = 6; + optional string jsonV = 7; + optional bytes compressedJsonV = 8; +} + message DeviceInfoProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; @@ -483,6 +516,10 @@ message ImageCacheKeyProto { optional string publicResourceKey = 2; } +message ToEdqsCoreServiceMsg { + bytes value = 1; +} + message LwM2MRegistrationRequestMsg { string tenantId = 1; string endpoint = 2; @@ -772,6 +809,76 @@ message DeviceInactivityProto { int64 lastInactivityTime = 5; } +message DeviceInactivityTimeoutUpdateProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 deviceIdMSB = 3; + int64 deviceIdLSB = 4; + int64 inactivityTimeout = 5; +} + +message CalculatedFieldTelemetryMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + string entityType = 3; + int64 entityIdMSB = 4; + int64 entityIdLSB = 5; + repeated CalculatedFieldIdProto previousCalculatedFields = 7; + repeated TsKvProto tsData = 9; + AttributeScopeProto scope = 10; + repeated AttributeValueProto attrData = 11; + repeated string removedTsKeys = 12; + repeated string removedAttrKeys = 13; + int64 tbMsgIdMSB = 14; + int64 tbMsgIdLSB = 15; + string tbMsgType = 16; +} + +message CalculatedFieldLinkedTelemetryMsgProto { + CalculatedFieldTelemetryMsgProto msg = 1; + repeated CalculatedFieldEntityCtxIdProto links = 2; +} + +message CalculatedFieldEntityCtxIdProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; + string entityType = 5; + int64 entityIdMSB = 6; + int64 entityIdLSB = 7; +} + +message CalculatedFieldIdProto { + int64 calculatedFieldIdMSB = 1; + int64 calculatedFieldIdLSB = 2; +} + +message SingleValueArgumentProto { + string argName = 1; + TsValueProto value = 2; + int64 version = 3; +} + +message TsDoubleValProto { + int64 ts = 1; + double value = 2; +} + +message TsRollingArgumentProto { + string key = 1; + int32 limit = 2; + int64 timeWindow = 3; + repeated TsDoubleValProto tsValue = 4; +} + +message CalculatedFieldStateProto { + CalculatedFieldEntityCtxIdProto id = 1; + string type = 2; + repeated SingleValueArgumentProto singleValueArguments = 3; + repeated TsRollingArgumentProto rollingValueArguments = 4; +} + //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; @@ -1015,7 +1122,11 @@ message TbAttributeDeleteProto { int64 tenantIdLSB = 5; string scope = 6; repeated string keys = 7; - bool notifyDevice = 8; + // DEPRECATED. FOR REMOVAL + // Since 4.0, this field is no longer used. + // Device notifications are now handled directly by DefaultTelemetrySubscriptionService, + // eliminating the need to pass this parameter through the queue and proto to DefaultSubscriptionManagerService. + optional bool notifyDevice = 8 [deprecated = true]; } message TbTimeSeriesDeleteProto { @@ -1127,6 +1238,13 @@ message ComponentLifecycleMsgProto { int64 entityIdMSB = 4; int64 entityIdLSB = 5; ComponentLifecycleEvent event = 6; + //Since 4.0. TODO: replace the DeviceNameOrTypeUpdateMsgProto + string oldName = 7; + string name = 8; + int64 oldProfileIdMSB = 9; + int64 oldProfileIdLSB = 10; + int64 profileIdMSB = 11; + int64 profileIdLSB = 12; } message EdgeEventMsgProto { @@ -1515,6 +1633,7 @@ message ToCoreMsg { DeviceConnectProto deviceConnectMsg = 50; DeviceDisconnectProto deviceDisconnectMsg = 51; DeviceInactivityProto deviceInactivityMsg = 52; + DeviceInactivityTimeoutUpdateProto deviceInactivityTimeoutUpdateMsg = 53; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ @@ -1533,6 +1652,7 @@ message ToCoreNotificationMsg { ToEdgeSyncRequestMsgProto toEdgeSyncRequest = 11 [deprecated = true]; FromEdgeSyncResponseMsgProto fromEdgeSyncResponse = 12 [deprecated = true]; ResourceCacheInvalidateMsg resourceCacheInvalidateMsg = 13; + ToEdqsCoreServiceMsg toEdqsCoreServiceMsg = 17; RestApiCallResponseMsgProto restApiCallResponseMsg = 50; } @@ -1553,6 +1673,17 @@ message ToEdgeEventNotificationMsg { EdgeEventMsgProto edgeEventMsg = 1; } +message ToCalculatedFieldMsg { + ComponentLifecycleMsgProto componentLifecycleMsg = 1; + CalculatedFieldTelemetryMsgProto telemetryMsg = 2; + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 3; +} + +message ToCalculatedFieldNotificationMsg { + ComponentLifecycleMsgProto componentLifecycleMsg = 1; + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 2; +} + /* Messages that are handled by ThingsBoard RuleEngine Service */ message ToRuleEngineMsg { int64 tenantIdMSB = 1; @@ -1661,3 +1792,33 @@ message HousekeeperTaskProto { int32 attempt = 50; repeated string errors = 51; } + +message ToEdqsMsg { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 customerIdMSB = 3; + int64 customerIdLSB = 4; + int64 ts = 5; + EdqsEventMsg eventMsg = 6; + EdqsRequestMsg requestMsg = 7; +} + +message FromEdqsMsg { + EdqsResponseMsg responseMsg = 1; +} + +message EdqsEventMsg { + string key = 1; + string objectType = 2; + bytes data = 3; + string eventType = 4; + optional int64 version = 5; +} + +message EdqsRequestMsg { + string value = 1; +} + +message EdqsResponseMsg { + string value = 1; +} diff --git a/common/queue/pom.xml b/common/queue/pom.xml index c0e51318d8..e28ebfbcf0 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -116,6 +116,10 @@ org.apache.curator curator-recipes + + org.xerial.snappy + snappy-java + org.springframework.boot spring-boot-starter-test diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 7a99be1997..7e7de64a5c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -31,7 +31,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; import static java.util.Collections.emptyList; @@ -95,9 +94,8 @@ public abstract class AbstractTbQueueConsumerTemplate i partitions = subscribeQueue.poll(); } if (!subscribed) { - List topicNames = getFullTopicNames(); - log.info("Subscribing to topics {}", topicNames); - doSubscribe(topicNames); + log.info("Subscribing to {}", partitions); + doSubscribe(partitions); subscribed = true; } records = partitions.isEmpty() ? emptyList() : doPoll(durationInMillis); @@ -120,9 +118,9 @@ public abstract class AbstractTbQueueConsumerTemplate i if (record != null) { result.add(decode(record)); } - } catch (IOException e) { - log.error("Failed decode record: [{}]", record); - throw new RuntimeException("Failed to decode record: ", e); + } catch (Exception e) { + log.error("Failed to decode record {}", record, e); + throw new RuntimeException("Failed to decode record " + record, e); } }); return result; @@ -149,6 +147,9 @@ public abstract class AbstractTbQueueConsumerTemplate i @Override public void commit() { if (consumerLock.isLocked()) { + if (stopped) { + return; + } log.error("commit. consumerLock is locked. will wait with no timeout. it looks like a race conditions or deadlock topic " + topic, new RuntimeException("stacktrace")); } consumerLock.lock(); @@ -166,7 +167,7 @@ public abstract class AbstractTbQueueConsumerTemplate i @Override public void unsubscribe() { - log.info("Unsubscribing and stopping consumer for topics {}", getFullTopicNames()); + log.info("Unsubscribing and stopping consumer for {}", partitions); stopped = true; consumerLock.lock(); try { @@ -187,7 +188,7 @@ public abstract class AbstractTbQueueConsumerTemplate i abstract protected T decode(R record) throws IOException; - abstract protected void doSubscribe(List topicNames); + abstract protected void doSubscribe(Set partitions); abstract protected void doCommit(); @@ -198,7 +199,9 @@ public abstract class AbstractTbQueueConsumerTemplate i if (partitions == null) { return Collections.emptyList(); } - return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + return partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .toList(); } protected boolean isLongPollingSupported() { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java index 71cf4bc1a5..55e07e5856 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java @@ -24,35 +24,36 @@ public class AbstractTbQueueTemplate { protected static final String RESPONSE_TOPIC_HEADER = "responseTopic"; protected static final String EXPIRE_TS_HEADER = "expireTs"; - protected byte[] uuidToBytes(UUID uuid) { + public static byte[] uuidToBytes(UUID uuid) { ByteBuffer buf = ByteBuffer.allocate(16); buf.putLong(uuid.getMostSignificantBits()); buf.putLong(uuid.getLeastSignificantBits()); return buf.array(); } - protected static UUID bytesToUuid(byte[] bytes) { + public static UUID bytesToUuid(byte[] bytes) { ByteBuffer bb = ByteBuffer.wrap(bytes); long firstLong = bb.getLong(); long secondLong = bb.getLong(); return new UUID(firstLong, secondLong); } - protected byte[] stringToBytes(String string) { + public static byte[] stringToBytes(String string) { return string.getBytes(StandardCharsets.UTF_8); } - protected String bytesToString(byte[] data) { + public static String bytesToString(byte[] data) { return new String(data, StandardCharsets.UTF_8); } - protected static byte[] longToBytes(long x) { + public static byte[] longToBytes(long x) { ByteBuffer longBuffer = ByteBuffer.allocate(Long.BYTES); longBuffer.putLong(0, x); return longBuffer.array(); } - protected static long bytesToLong(byte[] bytes) { + public static long bytesToLong(byte[] bytes) { return ByteBuffer.wrap(bytes).getLong(); } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 6a73155b39..4efb297491 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -211,6 +211,15 @@ public class DefaultTbQueueRequestTemplate send(Request request, long requestTimeoutNs) { + return send(request, requestTimeoutNs, null); + } + + @Override + public ListenableFuture send(Request request, Integer partition) { + return send(request, this.maxRequestTimeoutNs, partition); + } + + private ListenableFuture send(Request request, long requestTimeoutNs, Integer partition) { if (pendingRequests.mappingCount() >= maxPendingRequests) { log.warn("Pending request map is full [{}]! Consider to increase maxPendingRequests or increase processing performance. Request is {}", maxPendingRequests, request); return Futures.immediateFailedFuture(new RuntimeException("Pending request map is full!")); @@ -227,7 +236,7 @@ public class DefaultTbQueueRequestTemplate future, ResponseMetaData responseMetaData) { + void sendToRequestTemplate(Request request, UUID requestId, Integer partition, SettableFuture future, ResponseMetaData responseMetaData) { log.trace("[{}] Sending request, key [{}], expTime [{}], request {}", requestId, request.getKey(), responseMetaData.expTime, request); if (messagesStats != null) { messagesStats.incrementTotal(); } - requestTemplate.send(TopicPartitionInfo.builder().topic(requestTemplate.getDefaultTopic()).build(), request, new TbQueueCallback() { + TopicPartitionInfo tpi = TopicPartitionInfo.builder() + .topic(requestTemplate.getDefaultTopic()) + .partition(partition) + .useInternalPartition(partition != null) + .build(); + requestTemplate.send(tpi, request, new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { if (messagesStats != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java index 0c925e9334..abd254b269 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java @@ -28,6 +28,7 @@ import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueResponseTemplate; import java.util.List; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -77,9 +78,18 @@ public class DefaultTbQueueResponseTemplate handler) { - this.responseTemplate.init(); + public void subscribe() { requestTemplate.subscribe(); + } + + @Override + public void subscribe(Set partitions) { + requestTemplate.subscribe(partitions); + } + + @Override + public void launch(TbQueueHandler handler) { + this.responseTemplate.init(); loopExecutor.submit(() -> { while (!stopped) { try { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java index 4a5caa35b4..a5bf6c8861 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/TbProtoQueueMsg.java @@ -50,6 +50,7 @@ public class TbProtoQueueMsg i @Override public byte[] getData() { - return value.toByteArray(); + return value != null ? value.toByteArray() : null; } + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java similarity index 73% rename from application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java index ef1831c479..14394bbbe9 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue.consumer; +package org.thingsboard.server.queue.common.consumer; import lombok.Builder; import lombok.Getter; @@ -23,10 +23,9 @@ import org.thingsboard.server.common.data.queue.QueueConfig; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.UpdateConfigTask; +import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.UpdatePartitionsTask; import org.thingsboard.server.queue.discovery.QueueKey; -import org.thingsboard.server.service.queue.ruleengine.QueueEvent; -import org.thingsboard.server.service.queue.ruleengine.TbQueueConsumerManagerTask; -import org.thingsboard.server.service.queue.ruleengine.TbQueueConsumerTask; import java.util.Collection; import java.util.Collections; @@ -34,6 +33,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutorService; @@ -42,19 +42,24 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; import java.util.function.BiFunction; -import java.util.stream.Collectors; +import java.util.function.Consumer; @Slf4j public class MainQueueConsumerManager { + @Getter protected final QueueKey queueKey; @Getter protected C config; protected final MsgPackProcessor msgPackProcessor; protected final BiFunction> consumerCreator; + @Getter protected final ExecutorService consumerExecutor; + @Getter protected final ScheduledExecutorService scheduler; + @Getter protected final ExecutorService taskExecutor; + protected final Consumer uncaughtErrorHandler; private final java.util.Queue tasks = new ConcurrentLinkedQueue<>(); private final ReentrantLock lock = new ReentrantLock(); @@ -70,7 +75,8 @@ public class MainQueueConsumerManager> consumerCreator, ExecutorService consumerExecutor, ScheduledExecutorService scheduler, - ExecutorService taskExecutor) { + ExecutorService taskExecutor, + Consumer uncaughtErrorHandler) { this.queueKey = queueKey; this.config = config; this.msgPackProcessor = msgPackProcessor; @@ -78,6 +84,7 @@ public class MainQueueConsumerManager createConsumerWrapper(C config) { if (config.isConsumerPerPartition()) { - this.consumerWrapper = new ConsumerPerPartitionWrapper(); + return new ConsumerPerPartitionWrapper(); } else { - this.consumerWrapper = new SingleConsumerWrapper(); + return new SingleConsumerWrapper(); } - log.debug("[{}] Initialized consumer for queue: {}", queueKey, config); } public void update(C config) { - addTask(TbQueueConsumerManagerTask.configUpdate(config)); + addTask(new UpdateConfigTask(config)); } public void update(Set partitions) { - addTask(TbQueueConsumerManagerTask.partitionChange(partitions)); + addTask(new UpdatePartitionsTask(partitions)); } protected void addTask(TbQueueConsumerManagerTask todo) { @@ -123,10 +134,10 @@ public class MainQueueConsumerManager consumerLoop = consumerExecutor.submit(() -> { ThingsBoardThreadFactory.updateCurrentThreadName(consumerTask.getKey().toString()); + consumerLoop(consumerTask.getConsumer()); + log.info("[{}] Consumer stopped", consumerTask.getKey()); + try { - consumerLoop(consumerTask.getConsumer()); - } catch (Throwable e) { - log.error("Failure in consumer loop", e); + Runnable callback = consumerTask.getCallback(); + if (callback != null) { + callback.run(); + } + } catch (Throwable t) { + log.error("Failed to execute finish callback", t); } - log.info("[{}] Consumer stopped", consumerTask.getKey()); }); consumerTask.setTask(consumerLoop); } private void consumerLoop(TbQueueConsumer consumer) { - while (!stopped && !consumer.isStopped()) { - try { - List msgs = consumer.poll(config.getPollInterval()); - if (msgs.isEmpty()) { - continue; - } - processMsgs(msgs, consumer, config); - } catch (Exception e) { - if (!consumer.isStopped()) { - log.warn("Failed to process messages from queue", e); - try { - Thread.sleep(config.getPollInterval()); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); + try { + while (!stopped && !consumer.isStopped()) { + try { + List msgs = consumer.poll(config.getPollInterval()); + if (msgs.isEmpty()) { + continue; + } + processMsgs(msgs, consumer, config); + } catch (Exception e) { + if (!consumer.isStopped()) { + log.warn("Failed to process messages from queue", e); + try { + Thread.sleep(config.getPollInterval()); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } } } } - } - if (consumer.isStopped()) { + if (consumer.isStopped()) { + consumer.unsubscribe(); + } + } catch (Throwable t) { + log.error("Failure in consumer loop", t); + if (uncaughtErrorHandler != null) { + uncaughtErrorHandler.accept(t); + } consumer.unsubscribe(); } } protected void processMsgs(List msgs, TbQueueConsumer consumer, C config) throws Exception { + log.trace("Processing {} messages", msgs.size()); msgPackProcessor.process(msgs, consumer, config); + log.trace("Processed {} messages", msgs.size()); } public void stop() { @@ -236,13 +262,13 @@ public class MainQueueConsumerManager partitions) { - return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.joining(", ", "[", "]")); + private void awaitStop(int timeoutSec) { + log.debug("[{}] Waiting for consumers to stop", queueKey); + consumerWrapper.getConsumers().forEach(consumerTask -> consumerTask.awaitCompletion(timeoutSec)); + log.debug("[{}] Unsubscribed and stopped consumers", queueKey); } public interface MsgPackProcessor { @@ -267,15 +293,24 @@ public class MainQueueConsumerManager removedPartitions = new HashSet<>(consumers.keySet()); removedPartitions.removeAll(partitions); - log.info("[{}] Added partitions: {}, removed partitions: {}", queueKey, partitionsToString(addedPartitions), partitionsToString(removedPartitions)); - removedPartitions.forEach((tpi) -> consumers.get(tpi).initiateStop()); - removedPartitions.forEach((tpi) -> consumers.remove(tpi).awaitCompletion()); + log.info("[{}] Added partitions: {}, removed partitions: {}", queueKey, addedPartitions, removedPartitions); + removePartitions(removedPartitions); + addPartitions(addedPartitions, null); + } - addedPartitions.forEach((tpi) -> { + protected void removePartitions(Set removedPartitions) { + removedPartitions.forEach((tpi) -> Optional.ofNullable(consumers.get(tpi)).ifPresent(TbQueueConsumerTask::initiateStop)); + removedPartitions.forEach((tpi) -> Optional.ofNullable(consumers.remove(tpi)).ifPresent(TbQueueConsumerTask::awaitCompletion)); + } + + protected void addPartitions(Set partitions, Consumer onStop) { + partitions.forEach(tpi -> { Integer partitionId = tpi.getPartition().orElse(-1); String key = queueKey + "-" + partitionId; - TbQueueConsumerTask consumer = new TbQueueConsumerTask<>(key, () -> consumerCreator.apply(config, partitionId)); + Runnable callback = onStop != null ? () -> onStop.accept(tpi) : null; + + TbQueueConsumerTask consumer = new TbQueueConsumerTask<>(key, () -> consumerCreator.apply(config, partitionId), callback); consumers.put(tpi, consumer); consumer.subscribe(Set.of(tpi)); launchConsumer(consumer); @@ -293,7 +328,7 @@ public class MainQueueConsumerManager partitions) { - log.info("[{}] New partitions: {}", queueKey, partitionsToString(partitions)); + log.info("[{}] New partitions: {}", queueKey, partitions); if (partitions.isEmpty()) { if (consumer != null && consumer.isRunning()) { consumer.initiateStop(); @@ -304,7 +339,7 @@ public class MainQueueConsumerManager(queueKey, () -> consumerCreator.apply(config, null)); // no partitionId passed + consumer = new TbQueueConsumerTask<>(queueKey, () -> consumerCreator.apply(config, null), null); // no partitionId passed } consumer.subscribe(partitions); if (!consumer.isRunning()) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java new file mode 100644 index 0000000000..1d68cc9c4e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java @@ -0,0 +1,75 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common.consumer; + +import lombok.Builder; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.queue.QueueConfig; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.AddPartitionsTask; +import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.RemovePartitionsTask; +import org.thingsboard.server.queue.discovery.QueueKey; + +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; +import java.util.function.BiFunction; +import java.util.function.Consumer; + +@Slf4j +public class PartitionedQueueConsumerManager extends MainQueueConsumerManager { + + private final ConsumerPerPartitionWrapper consumerWrapper; + @Getter + private final String topic; + + @Builder(builderMethodName = "create") // not to conflict with super.builder() + public PartitionedQueueConsumerManager(QueueKey queueKey, String topic, long pollInterval, MsgPackProcessor msgPackProcessor, + BiFunction> consumerCreator, + ExecutorService consumerExecutor, ScheduledExecutorService scheduler, + ExecutorService taskExecutor, Consumer uncaughtErrorHandler) { + super(queueKey, QueueConfig.of(true, pollInterval), msgPackProcessor, consumerCreator, consumerExecutor, scheduler, taskExecutor, uncaughtErrorHandler); + this.topic = topic; + this.consumerWrapper = (ConsumerPerPartitionWrapper) super.consumerWrapper; + } + + @Override + protected void processTask(TbQueueConsumerManagerTask task) { + if (task instanceof AddPartitionsTask addPartitionsTask) { + log.info("[{}] Added partitions: {}", queueKey, addPartitionsTask.partitions()); + consumerWrapper.addPartitions(addPartitionsTask.partitions(), addPartitionsTask.onStop()); + } else if (task instanceof RemovePartitionsTask removePartitionsTask) { + log.info("[{}] Removed partitions: {}", queueKey, removePartitionsTask.partitions()); + consumerWrapper.removePartitions(removePartitionsTask.partitions()); + } + } + + public void addPartitions(Set partitions) { + addPartitions(partitions, null); + } + + public void addPartitions(Set partitions, Consumer onStop) { + addTask(new AddPartitionsTask(partitions, onStop)); + } + + public void removePartitions(Set partitions) { + addTask(new RemovePartitionsTask(partitions)); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java new file mode 100644 index 0000000000..8870ff2a2c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueStateService.java @@ -0,0 +1,98 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common.consumer; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueMsg; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTopic; + +@Slf4j +public class QueueStateService { + + private PartitionedQueueConsumerManager stateConsumer; + private PartitionedQueueConsumerManager eventConsumer; + + @Getter + private Set partitions; + private final Set partitionsInProgress = ConcurrentHashMap.newKeySet(); + private boolean initialized; + + private final ReadWriteLock partitionsLock = new ReentrantReadWriteLock(); + + public void init(PartitionedQueueConsumerManager stateConsumer, PartitionedQueueConsumerManager eventConsumer) { + this.stateConsumer = stateConsumer; + this.eventConsumer = eventConsumer; + } + + public void update(Set newPartitions) { + newPartitions = withTopic(newPartitions, stateConsumer.getTopic()); + var writeLock = partitionsLock.writeLock(); + writeLock.lock(); + Set oldPartitions = this.partitions != null ? this.partitions : Collections.emptySet(); + Set addedPartitions; + Set removedPartitions; + try { + addedPartitions = new HashSet<>(newPartitions); + addedPartitions.removeAll(oldPartitions); + removedPartitions = new HashSet<>(oldPartitions); + removedPartitions.removeAll(newPartitions); + this.partitions = newPartitions; + } finally { + writeLock.unlock(); + } + + if (!removedPartitions.isEmpty()) { + stateConsumer.removePartitions(removedPartitions); + eventConsumer.removePartitions(withTopic(removedPartitions, eventConsumer.getTopic())); + } + + if (!addedPartitions.isEmpty()) { + partitionsInProgress.addAll(addedPartitions); + stateConsumer.addPartitions(addedPartitions, partition -> { + var readLock = partitionsLock.readLock(); + readLock.lock(); + try { + partitionsInProgress.remove(partition); + log.info("Finished partition {} (still in progress: {})", partition, partitionsInProgress); + if (partitionsInProgress.isEmpty()) { + log.info("All partitions processed"); + } + if (this.partitions.contains(partition)) { + eventConsumer.addPartitions(Set.of(partition.withTopic(eventConsumer.getTopic()))); + } + } finally { + readLock.unlock(); + } + }); + } + initialized = true; + } + + public Set getPartitionsInProgress() { + return initialized ? partitionsInProgress : null; + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueTaskType.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueTaskType.java new file mode 100644 index 0000000000..93601146e9 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueTaskType.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common.consumer; + +import java.io.Serializable; + +public enum QueueTaskType implements Serializable { + + UPDATE_PARTITIONS, UPDATE_CONFIG, DELETE, + ADD_PARTITIONS, REMOVE_PARTITIONS + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerManagerTask.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerManagerTask.java new file mode 100644 index 0000000000..3380dd7e31 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerManagerTask.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common.consumer; + +import org.thingsboard.server.common.data.queue.QueueConfig; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; + +import java.util.Set; +import java.util.function.Consumer; + +public interface TbQueueConsumerManagerTask { + + QueueTaskType getType(); + + record DeleteQueueTask(boolean drainQueue) implements TbQueueConsumerManagerTask { + @Override + public QueueTaskType getType() { + return QueueTaskType.DELETE; + } + } + + record UpdateConfigTask(QueueConfig config) implements TbQueueConsumerManagerTask { + @Override + public QueueTaskType getType() { + return QueueTaskType.UPDATE_CONFIG; + } + } + + record UpdatePartitionsTask(Set partitions) implements TbQueueConsumerManagerTask { + @Override + public QueueTaskType getType() { + return QueueTaskType.UPDATE_PARTITIONS; + } + } + + record AddPartitionsTask(Set partitions, Consumer onStop) implements TbQueueConsumerManagerTask { + @Override + public QueueTaskType getType() { + return QueueTaskType.ADD_PARTITIONS; + } + } + + record RemovePartitionsTask(Set partitions) implements TbQueueConsumerManagerTask { + @Override + public QueueTaskType getType() { + return QueueTaskType.REMOVE_PARTITIONS; + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java similarity index 84% rename from application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java index 16642d5cf4..28066d9a91 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue.ruleengine; +package org.thingsboard.server.queue.common.consumer; import lombok.Getter; import lombok.Setter; @@ -35,14 +35,17 @@ public class TbQueueConsumerTask { private final Object key; private volatile TbQueueConsumer consumer; private volatile Supplier> consumerSupplier; + @Getter + private final Runnable callback; @Setter private Future task; - public TbQueueConsumerTask(Object key, Supplier> consumerSupplier) { + public TbQueueConsumerTask(Object key, Supplier> consumerSupplier, Runnable callback) { this.key = key; this.consumer = null; this.consumerSupplier = consumerSupplier; + this.callback = callback; } public TbQueueConsumer getConsumer() { @@ -70,13 +73,21 @@ public class TbQueueConsumerTask { } public void awaitCompletion() { + awaitCompletion(30); + } + + public void awaitCompletion(int timeoutSec) { log.trace("[{}] Awaiting finish", key); if (isRunning()) { try { - task.get(30, TimeUnit.SECONDS); + if (timeoutSec > 0) { + task.get(timeoutSec, TimeUnit.SECONDS); + } else { + task.get(); + } log.trace("[{}] Awaited finish", key); } catch (Exception e) { - log.warn("[{}] Failed to await for consumer to stop", key, e); + log.warn("[{}] Failed to await for consumer to stop (timeout {} sec)", key, timeoutSec, e); } task = null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index a2753b8832..609d3f8eee 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import org.thingsboard.server.queue.edqs.EdqsConfig; import org.thingsboard.server.queue.util.AfterContextReady; import java.net.InetAddress; @@ -39,12 +40,7 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; -import static org.thingsboard.common.util.SystemUtil.getCpuCount; -import static org.thingsboard.common.util.SystemUtil.getCpuUsage; -import static org.thingsboard.common.util.SystemUtil.getDiscSpaceUsage; -import static org.thingsboard.common.util.SystemUtil.getMemoryUsage; -import static org.thingsboard.common.util.SystemUtil.getTotalDiscSpace; -import static org.thingsboard.common.util.SystemUtil.getTotalMemory; +import static org.thingsboard.common.util.SystemUtil.*; @Component @@ -63,6 +59,9 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { @Value("${service.rule_engine.assigned_tenant_profiles:}") private Set assignedTenantProfiles; + @Autowired + private EdqsConfig edqsConfig; + @Autowired private ApplicationContext applicationContext; @@ -87,6 +86,11 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { if (!serviceTypes.contains(ServiceType.TB_RULE_ENGINE) || assignedTenantProfiles == null) { assignedTenantProfiles = Collections.emptySet(); } + if (serviceTypes.contains(ServiceType.EDQS)) { + if (StringUtils.isBlank(edqsConfig.getLabel())) { + edqsConfig.setLabel(serviceId); + } + } generateNewServiceInfoWithCurrentSystemInfo(); } @@ -123,6 +127,7 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { if (CollectionsUtil.isNotEmpty(assignedTenantProfiles)) { builder.addAllAssignedTenantProfiles(assignedTenantProfiles.stream().map(UUID::toString).collect(Collectors.toList())); } + builder.setLabel(edqsConfig.getLabel()); return serviceInfo = builder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index c86d9d1551..b42f8cc380 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -20,6 +20,7 @@ import com.google.common.hash.Hashing; import jakarta.annotation.PostConstruct; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; @@ -58,10 +59,16 @@ import static org.thingsboard.server.common.data.DataConstants.MAIN_QUEUE_NAME; @Slf4j public class HashPartitionService implements PartitionService { - @Value("${queue.core.topic}") + @Value("${queue.core.topic:tb_core}") private String coreTopic; @Value("${queue.core.partitions:10}") private Integer corePartitions; + @Value("${queue.calculated_fields.event_topic:tb_cf_event}") + private String cfEventTopic; + @Value("${queue.calculated_fields.state_topic:tb_cf_state}") + private String cfStateTopic; + @Value("${queue.calculated_fields.partitions:10}") + private Integer cfPartitions; @Value("${queue.vc.topic:tb_version_control}") private String vcTopic; @Value("${queue.vc.partitions:10}") @@ -70,6 +77,8 @@ public class HashPartitionService implements PartitionService { private String edgeTopic; @Value("${queue.edge.partitions:10}") private Integer edgePartitions; + @Value("${queue.edqs.partitions:12}") + private Integer edqsPartitions; @Value("${queue.partitions.hash_function_name:murmur3_128}") private String hashFunctionName; @@ -108,10 +117,16 @@ public class HashPartitionService implements PartitionService { @PostConstruct public void init() { this.hashFunction = forName(hashFunctionName); + QueueKey coreKey = new QueueKey(ServiceType.TB_CORE); partitionSizesMap.put(coreKey, corePartitions); partitionTopicsMap.put(coreKey, coreTopic); + partitionSizesMap.put(QueueKey.CF, cfPartitions); + partitionTopicsMap.put(QueueKey.CF, cfEventTopic); + partitionSizesMap.put(QueueKey.CF_STATES, cfPartitions); + partitionTopicsMap.put(QueueKey.CF_STATES, cfStateTopic); + QueueKey vcKey = new QueueKey(ServiceType.TB_VC_EXECUTOR); partitionSizesMap.put(vcKey, vcPartitions); partitionTopicsMap.put(vcKey, vcTopic); @@ -123,6 +138,10 @@ public class HashPartitionService implements PartitionService { QueueKey edgeKey = coreKey.withQueueName(EDGE_QUEUE_NAME); partitionSizesMap.put(edgeKey, edgePartitions); partitionTopicsMap.put(edgeKey, edgeTopic); + + QueueKey edqsKey = new QueueKey(ServiceType.EDQS); + partitionSizesMap.put(edqsKey, edqsPartitions); + partitionTopicsMap.put(edqsKey, "edqs"); // placeholder, not used } @AfterStartUp(order = AfterStartUp.QUEUE_INFO_INITIALIZATION) @@ -137,6 +156,11 @@ public class HashPartitionService implements PartitionService { return myPartitions.get(queueKey); } + @Override + public String getTopic(QueueKey queueKey) { + return partitionTopicsMap.get(queueKey); + } + private void doInitRuleEnginePartitions() { List queueRoutingInfoList = getQueueRoutingInfos(); queueRoutingInfoList.forEach(queue -> { @@ -211,7 +235,7 @@ public class HashPartitionService implements PartitionService { }); if (serviceInfoProvider.isService(ServiceType.TB_RULE_ENGINE)) { publishPartitionChangeEvent(ServiceType.TB_RULE_ENGINE, queueKeys.stream() - .collect(Collectors.toMap(k -> k, k -> Collections.emptySet()))); + .collect(Collectors.toMap(k -> k, k -> Collections.emptySet())), Collections.emptyMap()); } } @@ -312,7 +336,8 @@ public class HashPartitionService implements PartitionService { } } - private TopicPartitionInfo resolve(QueueKey queueKey, EntityId entityId) { + @Override + public TopicPartitionInfo resolve(QueueKey queueKey, EntityId entityId) { Integer partitionSize = partitionSizesMap.get(queueKey); if (partitionSize == null) { throw new IllegalStateException("Partitions info for queue " + queueKey + " is missing"); @@ -354,6 +379,11 @@ public class HashPartitionService implements PartitionService { } } + @Override + public boolean isSystemPartitionMine(ServiceType serviceType) { + return isMyPartition(serviceType, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID); + } + @Override public synchronized void recalculatePartitions(ServiceInfo currentService, List otherServices) { log.info("Recalculating partitions"); @@ -374,9 +404,9 @@ public class HashPartitionService implements PartitionService { partitionSizesMap.forEach((queueKey, size) -> { for (int i = 0; i < size; i++) { try { - ServiceInfo serviceInfo = resolveByPartitionIdx(queueServicesMap.get(queueKey), queueKey, i, responsibleServices); - log.trace("Server responsible for {}[{}] - {}", queueKey, i, serviceInfo != null ? serviceInfo.getServiceId() : "none"); - if (currentService.equals(serviceInfo)) { + List services = resolveByPartitionIdx(queueServicesMap.get(queueKey), queueKey, i, responsibleServices); + log.trace("Server responsible for {}[{}] - {}", queueKey, i, services); + if (services.contains(currentService)) { newPartitions.computeIfAbsent(queueKey, key -> new ArrayList<>()).add(i); } } catch (Exception e) { @@ -390,6 +420,7 @@ public class HashPartitionService implements PartitionService { myPartitions = newPartitions; Map> changedPartitionsMap = new HashMap<>(); + Map> oldPartitionsMap = new HashMap<>(); Set removed = new HashSet<>(); oldPartitions.forEach((queueKey, partitions) -> { @@ -410,19 +441,16 @@ public class HashPartitionService implements PartitionService { myPartitions.forEach((queueKey, partitions) -> { if (!partitions.equals(oldPartitions.get(queueKey))) { - Set tpiList = partitions.stream() - .map(partition -> buildTopicPartitionInfo(queueKey, partition)) - .collect(Collectors.toSet()); - changedPartitionsMap.put(queueKey, tpiList); + changedPartitionsMap.put(queueKey, toTpiList(queueKey, partitions)); + oldPartitionsMap.put(queueKey, toTpiList(queueKey, oldPartitions.get(queueKey))); } }); if (!changedPartitionsMap.isEmpty()) { - Map>> partitionsByServiceType = new HashMap<>(); - changedPartitionsMap.forEach((queueKey, partitions) -> { - partitionsByServiceType.computeIfAbsent(queueKey.getType(), serviceType -> new HashMap<>()) - .put(queueKey, partitions); - }); - partitionsByServiceType.forEach(this::publishPartitionChangeEvent); + changedPartitionsMap.entrySet().stream() + .collect(Collectors.groupingBy(entry -> entry.getKey().getType(), Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))) + .forEach((serviceType, partitionsMap) -> { + publishPartitionChangeEvent(serviceType, partitionsMap, oldPartitionsMap); + }); } if (currentOtherServices == null) { @@ -454,13 +482,15 @@ public class HashPartitionService implements PartitionService { applicationEventPublisher.publishEvent(new ServiceListChangedEvent(otherServices, currentService)); } - private void publishPartitionChangeEvent(ServiceType serviceType, Map> partitionsMap) { - log.info("Partitions changed: {}", System.lineSeparator() + partitionsMap.entrySet().stream() + private void publishPartitionChangeEvent(ServiceType serviceType, + Map> newPartitions, + Map> oldPartitions) { + log.info("Partitions changed: {}", System.lineSeparator() + newPartitions.entrySet().stream() .map(entry -> "[" + entry.getKey() + "] - [" + entry.getValue().stream() .map(tpi -> tpi.getPartition().orElse(-1).toString()).sorted() .collect(Collectors.joining(", ")) + "]") .collect(Collectors.joining(System.lineSeparator()))); - PartitionChangeEvent event = new PartitionChangeEvent(this, serviceType, partitionsMap); + PartitionChangeEvent event = new PartitionChangeEvent(this, serviceType, newPartitions, oldPartitions); try { applicationEventPublisher.publishEvent(event); } catch (Exception e) { @@ -468,6 +498,15 @@ public class HashPartitionService implements PartitionService { } } + private Set toTpiList(QueueKey queueKey, List partitions) { + if (partitions == null) { + return Collections.emptySet(); + } + return partitions.stream() + .map(partition -> buildTopicPartitionInfo(queueKey, partition)) + .collect(Collectors.toSet()); + } + @Override public Set getAllServiceIds(ServiceType serviceType) { return getAllServices(serviceType).stream().map(ServiceInfo::getServiceId).collect(Collectors.toSet()); @@ -496,7 +535,6 @@ public class HashPartitionService implements PartitionService { return result; } - @Override public int resolvePartitionIndex(UUID entityId, int partitions) { int hash = hash(entityId); @@ -514,6 +552,11 @@ public class HashPartitionService implements PartitionService { return list == null ? 0 : list.size(); } + @Override + public int getTotalCalculatedFieldPartitions() { + return cfPartitions; + } + private Map> getServiceKeyListMap(List services) { final Map> currentMap = new HashMap<>(); services.forEach(serviceInfo -> { @@ -598,6 +641,8 @@ public class HashPartitionService implements PartitionService { queueServiceList.computeIfAbsent(new QueueKey(serviceType).withQueueName(EDGE_QUEUE_NAME), key -> new ArrayList<>()).add(instance); } else if (ServiceType.TB_VC_EXECUTOR.equals(serviceType)) { queueServiceList.computeIfAbsent(new QueueKey(serviceType), key -> new ArrayList<>()).add(instance); + } else if (ServiceType.EDQS.equals(serviceType)) { + queueServiceList.computeIfAbsent(new QueueKey(serviceType), key -> new ArrayList<>()).add(instance); } } @@ -606,10 +651,11 @@ public class HashPartitionService implements PartitionService { } } - protected ServiceInfo resolveByPartitionIdx(List servers, QueueKey queueKey, int partition, - Map> responsibleServices) { + @NotNull + protected List resolveByPartitionIdx(List servers, QueueKey queueKey, int partition, + Map> responsibleServices) { if (servers == null || servers.isEmpty()) { - return null; + return Collections.emptyList(); } TenantId tenantId = queueKey.getTenantId(); @@ -637,15 +683,21 @@ public class HashPartitionService implements PartitionService { responsibleServices.put(profileId, responsible); } if (responsible.isEmpty()) { - return null; + return Collections.emptyList(); } servers = responsible; } int hash = hash(tenantId.getId()); - return servers.get(Math.abs((hash + partition) % servers.size())); + ServiceInfo server = servers.get(Math.abs((hash + partition) % servers.size())); + return server != null ? List.of(server) : Collections.emptyList(); + } else if (queueKey.getType() == ServiceType.EDQS) { + List> sets = servers.stream().collect(Collectors.groupingBy(ServiceInfo::getLabel)) + .entrySet().stream().sorted(Map.Entry.comparingByKey()).map(Map.Entry::getValue).toList(); + return sets.get(partition % sets.size()); } else { - return servers.get(partition % servers.size()); + ServiceInfo server = servers.get(partition % servers.size()); + return server != null ? List.of(server) : Collections.emptyList(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index 5fe97ae972..404b0258c0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -37,12 +37,18 @@ public interface PartitionService { TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); + TopicPartitionInfo resolve(QueueKey queueKey, EntityId entityId); + List resolveAll(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId); boolean isMyPartition(ServiceType serviceType, TenantId tenantId, EntityId entityId); + boolean isSystemPartitionMine(ServiceType serviceType); + List getMyPartitions(QueueKey queueKey); + String getTopic(QueueKey queueKey); + /** * Received from the Discovery service when network topology is changed. * @param currentService - current service information {@link org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo} @@ -61,8 +67,6 @@ public interface PartitionService { Set getOtherServices(ServiceType serviceType); - int resolvePartitionIndex(UUID entityId, int partitions); - void evictTenantInfo(TenantId tenantId); int countTransportsByType(String type); @@ -75,4 +79,8 @@ public interface PartitionService { boolean isManagedByCurrentService(TenantId tenantId); + int resolvePartitionIndex(UUID entityId, int partitions); + + int getTotalCalculatedFieldPartitions(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java index b06ba904c8..ca38959fdd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java @@ -23,6 +23,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; +import static org.thingsboard.server.common.data.DataConstants.CF_QUEUE_NAME; +import static org.thingsboard.server.common.data.DataConstants.CF_STATES_QUEUE_NAME; + @Data @AllArgsConstructor public class QueueKey { @@ -32,6 +35,9 @@ public class QueueKey { private final String queueName; private final TenantId tenantId; + public static final QueueKey CF = new QueueKey(ServiceType.TB_RULE_ENGINE).withQueueName(CF_QUEUE_NAME); + public static final QueueKey CF_STATES = new QueueKey(ServiceType.TB_RULE_ENGINE).withQueueName(CF_STATES_QUEUE_NAME); + public QueueKey(ServiceType type, Queue queue) { this.type = type; this.queueName = queue.getName(); @@ -56,6 +62,12 @@ public class QueueKey { this.tenantId = TenantId.SYS_TENANT_ID; } + public QueueKey(ServiceType type, String queueName) { + this.type = type; + this.queueName = queueName; + this.tenantId = TenantId.SYS_TENANT_ID; + } + @Override public String toString() { return "QK(" + queueName + "," + type + "," + diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java index f3b52cf23f..5992083d85 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java @@ -32,9 +32,28 @@ public class TopicService { @Value("${queue.prefix:}") private String prefix; + @Value("${queue.core.notifications-topic:tb_core.notifications}") + private String tbCoreNotificationsTopic; + + @Value("${queue.rule-engine.notifications-topic:tb_rule_engine.notifications}") + private String tbRuleEngineNotificationsTopic; + + @Value("${queue.transport.notifications-topics:tb_transport.notifications}") + private String tbTransportNotificationsTopic; + + @Value("${queue.edge.notifications-topic:tb_edge.notifications}") + private String tbEdgeNotificationsTopic; + + @Value("${queue.edge.event-notifications-topic:tb_edge_event.notifications}") + private String tbEdgeEventNotificationsTopic; + + @Value("${queue.calculated_fields.notifications-topic:calculated_field.notifications}") + private String tbCalculatedFieldNotificationsTopic; + private final ConcurrentMap tbCoreNotificationTopics = new ConcurrentHashMap<>(); private final ConcurrentMap tbRuleEngineNotificationTopics = new ConcurrentHashMap<>(); private final ConcurrentMap tbEdgeNotificationTopics = new ConcurrentHashMap<>(); + private final ConcurrentMap tbCalculatedFieldNotificationTopics = new ConcurrentHashMap<>(); private final ConcurrentReferenceHashMap tbEdgeEventsNotificationTopics = new ConcurrentReferenceHashMap<>(); /** @@ -47,19 +66,32 @@ public class TopicService { public TopicPartitionInfo getNotificationsTopic(ServiceType serviceType, String serviceId) { return switch (serviceType) { case TB_CORE -> tbCoreNotificationTopics.computeIfAbsent(serviceId, - id -> buildNotificationsTopicPartitionInfo(serviceType, serviceId)); + id -> buildNotificationsTopicPartitionInfo(tbCoreNotificationsTopic, serviceId)); case TB_RULE_ENGINE -> tbRuleEngineNotificationTopics.computeIfAbsent(serviceId, - id -> buildNotificationsTopicPartitionInfo(serviceType, serviceId)); - default -> buildNotificationsTopicPartitionInfo(serviceType, serviceId); + id -> buildNotificationsTopicPartitionInfo(tbRuleEngineNotificationsTopic, serviceId)); + case TB_TRANSPORT -> buildNotificationsTopicPartitionInfo(tbTransportNotificationsTopic, serviceId); + default -> throw new IllegalStateException("Unexpected service type: " + serviceType); }; } + private TopicPartitionInfo buildNotificationsTopicPartitionInfo(String topic, String serviceId) { + return buildTopicPartitionInfo(buildNotificationTopicName(topic, serviceId), null, null, false); + } + + public TopicPartitionInfo buildTopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean myPartition) { + return new TopicPartitionInfo(buildTopicName(topic), tenantId, partition, myPartition); + } + public TopicPartitionInfo getEdgeNotificationsTopic(String serviceId) { return tbEdgeNotificationTopics.computeIfAbsent(serviceId, id -> buildEdgeNotificationsTopicPartitionInfo(serviceId)); } private TopicPartitionInfo buildEdgeNotificationsTopicPartitionInfo(String serviceId) { - return buildTopicPartitionInfo("tb_edge.notifications." + serviceId, null, null, false); + return buildTopicPartitionInfo(buildNotificationTopicName(tbEdgeNotificationsTopic, serviceId), null, null, false); + } + + public TopicPartitionInfo getCalculatedFieldNotificationsTopic(String serviceId) { + return tbCalculatedFieldNotificationTopics.computeIfAbsent(serviceId, id -> buildNotificationsTopicPartitionInfo(tbCalculatedFieldNotificationsTopic, serviceId)); } public TopicPartitionInfo getEdgeEventNotificationsTopic(TenantId tenantId, EdgeId edgeId) { @@ -67,21 +99,17 @@ public class TopicService { } public TopicPartitionInfo buildEdgeEventNotificationsTopicPartitionInfo(TenantId tenantId, EdgeId edgeId) { - return buildTopicPartitionInfo("tb_edge_event.notifications." + tenantId + "." + edgeId, null, null, false); - } - - private TopicPartitionInfo buildNotificationsTopicPartitionInfo(ServiceType serviceType, String serviceId) { - return buildTopicPartitionInfo(serviceType.name().toLowerCase() + ".notifications." + serviceId, null, null, false); - } - - public TopicPartitionInfo buildTopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean myPartition) { - return new TopicPartitionInfo(buildTopicName(topic), tenantId, partition, myPartition); + return buildTopicPartitionInfo(tbEdgeEventNotificationsTopic + "." + tenantId + "." + edgeId, null, null, false); } public String buildTopicName(String topic) { return prefix.isBlank() ? topic : prefix + "." + topic; } + private String buildNotificationTopicName(String topic, String serviceId) { + return topic + "." + serviceId; + } + public String buildConsumerGroupId(String servicePrefix, TenantId tenantId, String queueName, Integer partitionId) { return this.buildTopicName( servicePrefix + queueName diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index 138e963188..f7a4d2abf6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -19,6 +19,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.ProtocolStringList; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; +import lombok.Getter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.curator.framework.CuratorFramework; @@ -68,6 +69,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi private Integer zkConnectionTimeout; @Value("${zk.session_timeout_ms}") private Integer zkSessionTimeout; + @Getter @Value("${zk.zk_dir}") private String zkDir; @Value("${zk.recalculate_delay:0}") @@ -80,6 +82,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi private final PartitionService partitionService; private ScheduledExecutorService zkExecutorService; + @Getter private CuratorFramework client; private PathChildrenCache cache; private String nodePath; @@ -140,6 +143,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi } else { log.info("Received application ready event. Starting current ZK node."); } + subscribeToEvents(); if (client.getState() != CuratorFrameworkState.STARTED) { log.debug("Ignoring application ready event, ZK client is not started, ZK client state [{}]", client.getState()); return; @@ -209,6 +213,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi try { destroyZkClient(); initZkClient(); + subscribeToEvents(); publishCurrentServer(); } catch (Exception e) { log.error("Failed to reconnect to ZK: {}", e.getMessage(), e); @@ -224,7 +229,6 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi client.start(); client.blockUntilConnected(); cache = new PathChildrenCache(client, zkNodesDir, true); - cache.getListenable().addListener(this); cache.start(); stopped = false; log.info("ZK client connected"); @@ -236,6 +240,10 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi } } + private void subscribeToEvents() { + cache.getListenable().addListener(this); + } + private void unpublishCurrentServer() { try { if (nodePath != null) { @@ -243,25 +251,21 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi } } catch (Exception e) { log.error("Failed to delete ZK node {}", nodePath, e); - throw new RuntimeException(e); } } private void destroyZkClient() { stopped = true; - try { - unpublishCurrentServer(); - } catch (Exception e) { - } + unpublishCurrentServer(); CloseableUtils.closeQuietly(cache); CloseableUtils.closeQuietly(client); log.info("ZK client disconnected"); } @PreDestroy - public void destroy() { - destroyZkClient(); + private void destroy() { zkExecutorService.shutdownNow(); + destroyZkClient(); log.info("Stopped discovery service"); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java index 530773a3c6..597463300a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java @@ -23,6 +23,8 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.discovery.QueueKey; import java.io.Serial; +import java.util.Collection; +import java.util.Collections; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -36,12 +38,17 @@ public class PartitionChangeEvent extends TbApplicationEvent { @Getter private final ServiceType serviceType; @Getter - private final Map> partitionsMap; + private final Map> newPartitions; + @Getter + private final Map> oldPartitions; - public PartitionChangeEvent(Object source, ServiceType serviceType, Map> partitionsMap) { + public PartitionChangeEvent(Object source, ServiceType serviceType, + Map> newPartitions, + Map> oldPartitions) { super(source); this.serviceType = serviceType; - this.partitionsMap = partitionsMap; + this.newPartitions = newPartitions; + this.oldPartitions = oldPartitions; } public Set getCorePartitions() { @@ -52,11 +59,20 @@ public class PartitionChangeEvent extends TbApplicationEvent { return getPartitionsByServiceTypeAndQueueName(ServiceType.TB_CORE, DataConstants.EDGE_QUEUE_NAME); } + public Set getPartitions() { + return newPartitions.values().stream().flatMap(Collection::stream).collect(Collectors.toSet()); + } + + public Set getCfPartitions() { + return newPartitions.getOrDefault(QueueKey.CF, Collections.emptySet()); + } + private Set getPartitionsByServiceTypeAndQueueName(ServiceType serviceType, String queueName) { - return partitionsMap.entrySet() + return newPartitions.entrySet() .stream() .filter(entry -> serviceType.equals(entry.getKey().getType()) && queueName.equals(entry.getKey().getQueueName())) .flatMap(entry -> entry.getValue().stream()) .collect(Collectors.toSet()); } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java new file mode 100644 index 0000000000..c3658d098a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}'=='true' && ('${service.type:null}'=='edqs' || " + + "(('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && " + + "'${queue.edqs.mode:null}'=='local'))") +public @interface EdqsComponent { +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java new file mode 100644 index 0000000000..e4e1e81815 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +@Data +public class EdqsConfig { + + @Value("${queue.edqs.partitions:12}") + private int partitions; + @Value("${service.edqs.label:}") + private String label; + @Value("#{'${queue.edqs.partitioning_strategy:tenant}'.toUpperCase()}") + private EdqsPartitioningStrategy partitioningStrategy; + + @Value("${queue.edqs.requests_topic:edqs.requests}") + private String requestsTopic; + @Value("${queue.edqs.responses_topic:edqs.responses}") + private String responsesTopic; + @Value("${queue.edqs.poll_interval:125}") + private long pollInterval; + @Value("${queue.edqs.max_pending_requests:10000}") + private int maxPendingRequests; + @Value("${queue.edqs.max_request_timeout:20000}") + private int maxRequestTimeout; + + public String getLabel() { + if (partitioningStrategy == EdqsPartitioningStrategy.NONE) { + label = "all"; // single set for all instances, so that each instance has all partitions + } + return label; + } + + public enum EdqsPartitioningStrategy { + TENANT, NONE + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java new file mode 100644 index 0000000000..d859b50994 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import lombok.Getter; + +@Getter +public enum EdqsQueue { + + EVENTS("edqs.events", false, false), + STATE("edqs.state", true, true); + + private final String topic; + private final boolean readFromBeginning; + private final boolean stopWhenRead; + + EdqsQueue(String topic, boolean readFromBeginning, boolean stopWhenRead) { + this.topic = topic; + this.readFromBeginning = readFromBeginning; + this.stopWhenRead = stopWhenRead; + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java new file mode 100644 index 0000000000..fed786e120 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +public interface EdqsQueueFactory { + + TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue); + + TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue, String group); + + TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue); + + TbQueueResponseTemplate, TbProtoQueueMsg> createEdqsResponseTemplate(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java new file mode 100644 index 0000000000..e414d24fd9 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}'=='true' && '${service.type:null}'=='monolith' && '${queue.edqs.mode:null}'=='local' && '${queue.type:null}'=='in-memory'") +public @interface InMemoryEdqsComponent { +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java new file mode 100644 index 0000000000..0b6cc1909d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java @@ -0,0 +1,79 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.common.stats.StatsType; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.memory.InMemoryStorage; +import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; +import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; + +@Component +@InMemoryEdqsComponent +@RequiredArgsConstructor +public class InMemoryEdqsQueueFactory implements EdqsQueueFactory { + + private final InMemoryStorage storage; + private final EdqsConfig edqsConfig; + private final StatsFactory statsFactory; + + @Override + public TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue) { + if (queue == EdqsQueue.STATE) { + throw new UnsupportedOperationException(); + } + return new InMemoryTbQueueConsumer<>(storage, queue.getTopic()); + } + + @Override + public TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue, String group) { + return createEdqsMsgConsumer(queue); + } + + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + if (queue == EdqsQueue.STATE) { + throw new UnsupportedOperationException(); + } + return new InMemoryTbQueueProducer<>(storage, queue.getTopic()); + } + + @Override + public TbQueueResponseTemplate, TbProtoQueueMsg> createEdqsResponseTemplate() { + TbQueueConsumer> requestConsumer = new InMemoryTbQueueConsumer<>(storage, edqsConfig.getRequestsTopic()); + TbQueueProducer> responseProducer = new InMemoryTbQueueProducer<>(storage, edqsConfig.getResponsesTopic()); + return DefaultTbQueueResponseTemplate., TbProtoQueueMsg>builder() + .requestTemplate(requestConsumer) + .responseTemplate(responseProducer) + .maxPendingRequests(edqsConfig.getMaxPendingRequests()) + .requestTimeout(edqsConfig.getMaxRequestTimeout()) + .pollInterval(edqsConfig.getPollInterval()) + .stats(statsFactory.createMessagesStats(StatsType.EDQS.getName())) + .executor(ThingsBoardExecutors.newWorkStealingPool(5, "edqs")) + .build(); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java new file mode 100644 index 0000000000..3a2b282724 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}'=='true' && ('${service.type:null}'=='edqs' || " + + "(('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && " + + "'${queue.edqs.mode:null}'=='local' && '${queue.type:null}'=='kafka'))") +public @interface KafkaEdqsComponent { +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java new file mode 100644 index 0000000000..42ca604841 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java @@ -0,0 +1,129 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.common.stats.StatsType; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; + +import java.util.concurrent.atomic.AtomicInteger; + +@Component +@KafkaEdqsComponent +public class KafkaEdqsQueueFactory implements EdqsQueueFactory { + + private final TbKafkaSettings kafkaSettings; + private final TbKafkaAdmin edqsEventsAdmin; + private final TbKafkaAdmin edqsRequestsAdmin; + private final TbKafkaAdmin edqsStateAdmin; + private final EdqsConfig edqsConfig; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbKafkaConsumerStatsService consumerStatsService; + private final TopicService topicService; + private final StatsFactory statsFactory; + + private final AtomicInteger consumerCounter = new AtomicInteger(); + + public KafkaEdqsQueueFactory(TbKafkaSettings kafkaSettings, TbKafkaTopicConfigs topicConfigs, + EdqsConfig edqsConfig, TbServiceInfoProvider serviceInfoProvider, + TbKafkaConsumerStatsService consumerStatsService, TopicService topicService, + StatsFactory statsFactory) { + this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, topicConfigs.getEdqsEventsConfigs()); + this.edqsRequestsAdmin = new TbKafkaAdmin(kafkaSettings, topicConfigs.getEdqsRequestsConfigs()); + this.edqsStateAdmin = new TbKafkaAdmin(kafkaSettings, topicConfigs.getEdqsStateConfigs()); + this.kafkaSettings = kafkaSettings; + this.edqsConfig = edqsConfig; + this.serviceInfoProvider = serviceInfoProvider; + this.consumerStatsService = consumerStatsService; + this.topicService = topicService; + this.statsFactory = statsFactory; + } + + @Override + public TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue) { + String consumerGroup = "edqs-" + queue.name().toLowerCase() + "-consumer-group-" + serviceInfoProvider.getServiceId(); + return createEdqsMsgConsumer(queue, consumerGroup); + } + + @Override + public TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue, String group) { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(queue.getTopic())) + .readFromBeginning(queue.isReadFromBeginning()) + .stopWhenRead(queue.isStopWhenRead()) + .clientId("edqs-" + queue.name().toLowerCase() + "-" + consumerCounter.getAndIncrement() + "-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName(group)) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdqsMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(queue == EdqsQueue.STATE ? edqsStateAdmin : edqsEventsAdmin) + .statsService(consumerStatsService) + .build(); + } + + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + return TbKafkaProducerTemplate.>builder() + .clientId("edqs-" + queue.name().toLowerCase() + "-producer-" + serviceInfoProvider.getServiceId()) + .settings(kafkaSettings) + .admin(queue == EdqsQueue.STATE ? edqsStateAdmin : edqsEventsAdmin) + .build(); + } + + @Override + public TbQueueResponseTemplate, TbProtoQueueMsg> createEdqsResponseTemplate() { + String requestsConsumerGroup = "edqs-requests-consumer-group-" + edqsConfig.getLabel(); + var requestConsumer = TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(edqsConfig.getRequestsTopic())) + .clientId("edqs-requests-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName(requestsConsumerGroup)) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToEdqsMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(edqsRequestsAdmin) + .statsService(consumerStatsService); + var responseProducer = TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("edqs-response-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(edqsConfig.getResponsesTopic())) + .admin(edqsRequestsAdmin); + return DefaultTbQueueResponseTemplate., TbProtoQueueMsg>builder() + .requestTemplate(requestConsumer.build()) + .responseTemplate(responseProducer.build()) + .maxPendingRequests(edqsConfig.getMaxPendingRequests()) + .requestTimeout(edqsConfig.getMaxRequestTimeout()) + .pollInterval(edqsConfig.getPollInterval()) + .stats(statsFactory.createMessagesStats(StatsType.EDQS.getName())) + .executor(ThingsBoardExecutors.newWorkStealingPool(5, "edqs")) + .build(); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLock.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLock.java new file mode 100644 index 0000000000..e5553078e4 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLock.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.environment; + +public interface DistributedLock { + + void lock(); + + void unlock(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLockService.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLockService.java new file mode 100644 index 0000000000..3fe5d7b21b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLockService.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.environment; + +public interface DistributedLockService { + + DistributedLock getLock(String key); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/environment/DummyDistributedLockService.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DummyDistributedLockService.java new file mode 100644 index 0000000000..96483c1f5a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DummyDistributedLockService.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.environment; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import java.util.concurrent.locks.ReentrantLock; + +@Service +@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) +public class DummyDistributedLockService implements DistributedLockService { + + @Override + public DistributedLock getLock(String key) { + return new DummyDistributedLock(); + } + + @RequiredArgsConstructor + private static class DummyDistributedLock implements DistributedLock { + + private final ReentrantLock lock = new ReentrantLock(); + + @SneakyThrows + @Override + public void lock() { + lock.lock(); + } + + @SneakyThrows + @Override + public void unlock() { + lock.unlock(); + } + + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/environment/ZkDistributedLockService.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/ZkDistributedLockService.java new file mode 100644 index 0000000000..65405c3d7e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/ZkDistributedLockService.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.environment; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.curator.framework.recipes.locks.InterProcessLock; +import org.apache.curator.framework.recipes.locks.InterProcessMutex; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.discovery.ZkDiscoveryService; + +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "true") +@Slf4j +public class ZkDistributedLockService implements DistributedLockService { + + private final ZkDiscoveryService zkDiscoveryService; + + @Override + public DistributedLock getLock(String key) { + return new ZkDistributedLock(key); + } + + @RequiredArgsConstructor + private class ZkDistributedLock implements DistributedLock { + + private final InterProcessLock interProcessLock; + + public ZkDistributedLock(String key) { + this.interProcessLock = new InterProcessMutex(zkDiscoveryService.getClient(), zkDiscoveryService.getZkDir() + "/locks/" + key); + } + + @SneakyThrows + @Override + public void lock() { + interProcessLock.acquire(); + } + + @SneakyThrows + @Override + public void unlock() { + interProcessLock.release(); + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java index 99bd2c8401..ffd4321060 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java @@ -23,12 +23,19 @@ import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders; import java.util.UUID; public class KafkaTbQueueMsg implements TbQueueMsg { + + private static final int UUID_LENGTH = 36; + private final UUID key; private final TbQueueMsgHeaders headers; private final byte[] data; public KafkaTbQueueMsg(ConsumerRecord record) { - this.key = UUID.fromString(record.key()); + if (record.key().length() <= UUID_LENGTH) { + this.key = UUID.fromString(record.key()); + } else { + this.key = UUID.randomUUID(); + } TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); record.headers().forEach(header -> { headers.put(header.key(), header.value()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java index e774bd74a7..f393a27ddf 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java @@ -57,7 +57,6 @@ public class TbKafkaAdmin implements TbQueueAdmin { String numPartitionsStr = topicConfigs.get(TbKafkaTopicConfigs.NUM_PARTITIONS_SETTING); if (numPartitionsStr != null) { numPartitions = Integer.parseInt(numPartitionsStr); - topicConfigs.remove("partitions"); } else { numPartitions = 1; } @@ -71,7 +70,9 @@ public class TbKafkaAdmin implements TbQueueAdmin { return; } try { - NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor).configs(PropertyUtils.getProps(topicConfigs, properties)); + Map configs = PropertyUtils.getProps(topicConfigs, properties); + configs.remove(TbKafkaTopicConfigs.NUM_PARTITIONS_SETTING); + NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor).configs(configs); createTopic(newTopic).values().get(topic).get(); topics.add(topic); } catch (ExecutionException ee) { @@ -188,6 +189,9 @@ public class TbKafkaAdmin implements TbQueueAdmin { public boolean isTopicEmpty(String topic) { try { + if (!getTopics().contains(topic)) { + return true; + } TopicDescription topicDescription = settings.getAdminClient().describeTopics(Collections.singletonList(topic)).topicNameValues().get(topic).get(); List partitions = topicDescription.partitions().stream().map(partitionInfo -> new TopicPartition(topic, partitionInfo.partition())).toList(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java index 3879d2cffd..9d1088e188 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java @@ -26,15 +26,10 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.queue.discovery.PartitionService; import java.time.Duration; import java.util.ArrayList; @@ -56,10 +51,6 @@ public class TbKafkaConsumerStatsService { private final TbKafkaSettings kafkaSettings; private final TbKafkaConsumerStatisticConfig statsConfig; - @Lazy - @Autowired - private PartitionService partitionService; - private Consumer consumer; private ScheduledExecutorService statsPrintScheduler; @@ -111,9 +102,7 @@ public class TbKafkaConsumerStatsService { } private boolean isStatsPrintRequired() { - boolean isMyRuleEnginePartition = partitionService.isMyPartition(ServiceType.TB_RULE_ENGINE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID); - boolean isMyCorePartition = partitionService.isMyPartition(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID); - return log.isInfoEnabled() && (isMyRuleEnginePartition || isMyCorePartition); + return log.isInfoEnabled(); } private List getTopicsStatsWithLag(Map groupOffsets, Map endOffsets) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index 02bc408c6e..d219428941 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -18,10 +18,13 @@ package org.thingsboard.server.queue.kafka; import lombok.Builder; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.TopicPartition; import org.springframework.util.StopWatch; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; @@ -29,9 +32,14 @@ import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; import java.io.IOException; import java.time.Duration; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; /** * Created by ashvayka on 24.09.18. @@ -46,10 +54,16 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue private final TbKafkaConsumerStatsService statsService; private final String groupId; + private final boolean readFromBeginning; // reset offset to beginning + private final boolean stopWhenRead; // stop consuming when reached end offset remembered on start + private int readCount; + private Map endOffsets; // needed if stopWhenRead is true + @Builder private TbKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, String clientId, String groupId, String topic, - TbQueueAdmin admin, TbKafkaConsumerStatsService statsService) { + TbQueueAdmin admin, TbKafkaConsumerStatsService statsService, + boolean readFromBeginning, boolean stopWhenRead) { super(topic); Properties props = settings.toConsumerProps(topic); props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); @@ -67,13 +81,55 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue this.admin = admin; this.consumer = new KafkaConsumer<>(props); this.decoder = decoder; + this.readFromBeginning = readFromBeginning; + this.stopWhenRead = stopWhenRead; } @Override - protected void doSubscribe(List topicNames) { - if (!topicNames.isEmpty()) { - topicNames.forEach(admin::createTopicIfNotExists); - consumer.subscribe(topicNames); + protected void doSubscribe(Set partitions) { + Map> topics; + if (partitions == null) { + topics = Collections.emptyMap(); + } else { + topics = new HashMap<>(); + partitions.forEach(tpi -> { + if (tpi.isUseInternalPartition()) { + topics.computeIfAbsent(tpi.getFullTopicName(), t -> new ArrayList<>()).add(tpi.getPartition().get()); + } else { + topics.put(tpi.getFullTopicName(), null); + } + }); + } + if (!topics.isEmpty()) { + topics.keySet().forEach(admin::createTopicIfNotExists); + List toSubscribe = new ArrayList<>(); + topics.forEach((topic, kafkaPartitions) -> { + if (kafkaPartitions == null) { + toSubscribe.add(topic); + } else { + List topicPartitions = kafkaPartitions.stream() + .map(partition -> new TopicPartition(topic, partition)) + .toList(); + consumer.assign(topicPartitions); + onPartitionsAssigned(topicPartitions); + } + }); + if (!toSubscribe.isEmpty()) { + if (readFromBeginning || stopWhenRead) { + consumer.subscribe(toSubscribe, new ConsumerRebalanceListener() { + @Override + public void onPartitionsRevoked(Collection partitions) {} + + @Override + public void onPartitionsAssigned(Collection partitions) { + log.debug("Handling onPartitionsAssigned {}", partitions); + TbKafkaConsumerTemplate.this.onPartitionsAssigned(partitions); + } + }); + } else { + consumer.subscribe(toSubscribe); + } + } } else { log.info("unsubscribe due to empty topic list"); consumer.unsubscribe(); @@ -92,12 +148,43 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue stopWatch.stop(); log.trace("poll topic {} took {}ms", getTopic(), stopWatch.getTotalTimeMillis()); + List> recordList; if (records.isEmpty()) { - return Collections.emptyList(); + recordList = Collections.emptyList(); } else { - List> recordList = new ArrayList<>(256); - records.forEach(recordList::add); - return recordList; + recordList = new ArrayList<>(256); + records.forEach(record -> { + recordList.add(record); + if (stopWhenRead && endOffsets != null) { + readCount++; + int partition = record.partition(); + Long endOffset = endOffsets.get(partition); + if (endOffset == null) { + log.warn("End offset not found for {} [{}]", record.topic(), partition); + return; + } + log.trace("[{}-{}] Got record offset {}, expected end offset: {}", record.topic(), partition, record.offset(), endOffset - 1); + if (record.offset() >= endOffset - 1) { + endOffsets.remove(partition); + } + } + }); + } + if (stopWhenRead && endOffsets != null && endOffsets.isEmpty()) { + log.info("Finished reading {}, processed {} messages", partitions, readCount); + stop(); + } + return recordList; + } + + private void onPartitionsAssigned(Collection partitions) { + if (readFromBeginning) { + consumer.seekToBeginning(partitions); + } + if (stopWhenRead) { + endOffsets = consumer.endOffsets(partitions).entrySet().stream() + .filter(entry -> entry.getValue() > 0) + .collect(Collectors.toMap(entry -> entry.getKey().partition(), Map.Entry::getValue)); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java index 6a303fc05a..cac6f2ea1e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java @@ -54,7 +54,7 @@ public class TbKafkaProducerTemplate implements TbQueuePro private final TbQueueAdmin admin; - private final Set topics; + private final Set topics; @Getter private final String clientId; @@ -97,16 +97,21 @@ public class TbKafkaProducerTemplate implements TbQueuePro @Override public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + send(tpi, msg.getKey().toString(), msg, callback); + } + + public void send(TopicPartitionInfo tpi, String key, T msg, TbQueueCallback callback) { try { - createTopicIfNotExist(tpi); - String key = msg.getKey().toString(); + String topic = tpi.getFullTopicName(); + createTopicIfNotExist(topic); byte[] data = msg.getData(); ProducerRecord record; List

headers = msg.getHeaders().getData().entrySet().stream().map(e -> new RecordHeader(e.getKey(), e.getValue())).collect(Collectors.toList()); if (log.isDebugEnabled()) { addAnalyticHeaders(headers); } - record = new ProducerRecord<>(tpi.getFullTopicName(), null, key, data, headers); + Integer partition = tpi.isUseInternalPartition() ? tpi.getPartition().orElse(null) : null; + record = new ProducerRecord<>(topic, partition, key, data, headers); producer.send(record, (metadata, exception) -> { if (exception == null) { if (callback != null) { @@ -116,7 +121,7 @@ public class TbKafkaProducerTemplate implements TbQueuePro if (callback != null) { callback.onFailure(exception); } else { - log.warn("Producer template failure: {}", exception.getMessage(), exception); + log.warn("Producer template failure", exception); } } }); @@ -130,12 +135,12 @@ public class TbKafkaProducerTemplate implements TbQueuePro } } - private void createTopicIfNotExist(TopicPartitionInfo tpi) { - if (topics.contains(tpi)) { + private void createTopicIfNotExist(String topic) { + if (topics.contains(topic)) { return; } - admin.createTopicIfNotExists(tpi.getFullTopicName()); - topics.add(tpi); + admin.createTopicIfNotExists(topic); + topics.add(topic); } @Override @@ -144,4 +149,5 @@ public class TbKafkaProducerTemplate implements TbQueuePro producer.close(); } } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index 5b40b50805..aebda5a5bc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -52,6 +52,16 @@ public class TbKafkaTopicConfigs { private String housekeeperProperties; @Value("${queue.kafka.topic-properties.housekeeper-reprocessing:}") private String housekeeperReprocessingProperties; + @Value("${queue.kafka.topic-properties.calculated-field:}") + private String calculatedFieldProperties; + @Value("${queue.kafka.topic-properties.calculated-field-state:}") + private String calculatedFieldStateProperties; + @Value("${queue.kafka.topic-properties.edqs-events:}") + private String edqsEventsProperties; + @Value("${queue.kafka.topic-properties.edqs-requests:}") + private String edqsRequestsProperties; + @Value("${queue.kafka.topic-properties.edqs-state:}") + private String edqsStateProperties; @Getter private Map coreConfigs; @@ -79,6 +89,16 @@ public class TbKafkaTopicConfigs { private Map edgeConfigs; @Getter private Map edgeEventConfigs; + @Getter + private Map calculatedFieldConfigs; + @Getter + private Map calculatedFieldStateConfigs; + @Getter + private Map edqsEventsConfigs; + @Getter + private Map edqsRequestsConfigs; + @Getter + private Map edqsStateConfigs; @PostConstruct private void init() { @@ -97,6 +117,11 @@ public class TbKafkaTopicConfigs { housekeeperReprocessingConfigs = PropertyUtils.getProps(housekeeperReprocessingProperties); edgeConfigs = PropertyUtils.getProps(edgeProperties); edgeEventConfigs = PropertyUtils.getProps(edgeEventProperties); + calculatedFieldConfigs = PropertyUtils.getProps(calculatedFieldProperties); + calculatedFieldStateConfigs = PropertyUtils.getProps(calculatedFieldStateProperties); + edqsEventsConfigs = PropertyUtils.getProps(edqsEventsProperties); + edqsRequestsConfigs = PropertyUtils.getProps(edqsRequestsProperties); + edqsStateConfigs = PropertyUtils.getProps(edqsStateProperties); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/EdqsClientQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/EdqsClientQueueFactory.java new file mode 100644 index 0000000000..95be49f82b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/EdqsClientQueueFactory.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.edqs.EdqsQueue; + +public interface EdqsClientQueueFactory { + + TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue); + + TbQueueRequestTemplate, TbProtoQueueMsg> createEdqsRequestTemplate(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index e7ba2fed6d..e97af10ecc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.provider; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.scheduling.annotation.Scheduled; @@ -23,16 +24,24 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.memory.InMemoryStorage; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; @@ -43,37 +52,22 @@ import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; @Slf4j @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && '${service.type:null}'=='monolith'") +@RequiredArgsConstructor public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { private final TopicService topicService; private final TbQueueCoreSettings coreSettings; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin queueAdmin; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueVersionControlSettings vcSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; + private final TbQueueCalculatedFieldSettings calculatedFieldSettings; + private final EdqsConfig edqsConfig; private final InMemoryStorage storage; - public InMemoryMonolithQueueFactory(TopicService topicService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueVersionControlSettings vcSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbQueueEdgeSettings edgeSettings, - InMemoryStorage storage) { - this.topicService = topicService; - this.coreSettings = coreSettings; - this.vcSettings = vcSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.edgeSettings = edgeSettings; - this.storage = storage; - } - @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic())); @@ -139,6 +133,31 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return null; } + @Override + public TbQueueConsumer> createToCalculatedFieldMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + } + + @Override + public TbQueueProducer> createToCalculatedFieldMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + } + + @Override + public TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.getCalculatedFieldNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); + } + + @Override + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); + } + + @Override + public TbQueueProducer> createCalculatedFieldStateProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); + } + @Override public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); @@ -209,6 +228,31 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return null; } + @Override + public TbQueueProducer> createToCalculatedFieldNotificationMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + } + + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + return new InMemoryTbQueueProducer<>(storage, queue.getTopic()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createEdqsRequestTemplate() { + TbQueueProducer> requestProducer = new InMemoryTbQueueProducer<>(storage, edqsConfig.getRequestsTopic()); + TbQueueConsumer> responseConsumer = new InMemoryTbQueueConsumer<>(storage, edqsConfig.getResponsesTopic()); + + return DefaultTbQueueRequestTemplate., TbProtoQueueMsg>builder() + .queueAdmin(queueAdmin) + .requestTemplate(requestProducer) + .responseTemplate(responseConsumer) + .maxPendingRequests(edqsConfig.getMaxPendingRequests()) + .maxRequestTimeout(edqsConfig.getMaxRequestTimeout()) + .pollInterval(edqsConfig.getPollInterval()) + .build(); + } + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { storage.printStats(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index f1210c4d40..bb9c1a028e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -25,11 +25,16 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -48,12 +53,15 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; @@ -79,7 +87,9 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueVersionControlSettings vcSettings; private final TbQueueEdgeSettings edgeSettings; + private final TbQueueCalculatedFieldSettings calculatedFieldSettings; private final TbKafkaConsumerStatsService consumerStatsService; + private final EdqsConfig edqsConfig; private final TbQueueAdmin coreAdmin; private final TbKafkaAdmin ruleEngineAdmin; @@ -94,6 +104,10 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin housekeeperReprocessingAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin cfAdmin; + private final TbQueueAdmin cfStateAdmin; + private final TbQueueAdmin edqsEventsAdmin; + private final TbKafkaAdmin edqsRequestsAdmin; private final AtomicLong consumerCount = new AtomicLong(); private final AtomicLong edgeConsumerCount = new AtomicLong(); @@ -107,8 +121,10 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbQueueVersionControlSettings vcSettings, TbQueueEdgeSettings edgeSettings, + TbQueueCalculatedFieldSettings calculatedFieldSettings, TbKafkaConsumerStatsService consumerStatsService, - TbKafkaTopicConfigs kafkaTopicConfigs) { + TbKafkaTopicConfigs kafkaTopicConfigs, + EdqsConfig edqsConfig) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -120,6 +136,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.vcSettings = vcSettings; this.consumerStatsService = consumerStatsService; this.edgeSettings = edgeSettings; + this.calculatedFieldSettings = calculatedFieldSettings; + this.edqsConfig = edqsConfig; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -134,6 +152,10 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); + this.cfStateAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldStateConfigs()); + this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsEventsConfigs()); + this.edqsRequestsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsRequestsConfigs()); } @Override @@ -491,6 +513,113 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return requestBuilder.build(); } + @Override + public TbQueueConsumer> createToCalculatedFieldMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + consumerBuilder.clientId("monolith-calculated-field-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.groupId(topicService.buildTopicName("monolith-calculated-field-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(cfAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToCalculatedFieldMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-calculated-field-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(cfAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.getCalculatedFieldNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("monolith-calculated-field-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId(topicService.buildTopicName("monolith-calculated-field-notifications-consumer-" + serviceInfoProvider.getServiceId())); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToCalculatedFieldNotificationMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-calculated-field-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(notificationAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())) + .readFromBeginning(true) + .stopWhenRead(true) + .clientId("monolith-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()) + .groupId(topicService.buildTopicName("monolith-calculated-field-state-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), msg.getData() != null ? CalculatedFieldStateProto.parseFrom(msg.getData()) : null, msg.getHeaders())) + .admin(cfStateAdmin) + .statsService(consumerStatsService) + .build(); + } + + @Override + public TbQueueProducer> createCalculatedFieldStateProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("monolith-calculated-field-state-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())) + .admin(cfStateAdmin) + .build(); + } + + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + return TbKafkaProducerTemplate.>builder() + .clientId("edqs-producer-" + queue.name().toLowerCase() + "-" + serviceInfoProvider.getServiceId()) + .settings(kafkaSettings) + .admin(edqsEventsAdmin) + .build(); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createEdqsRequestTemplate() { + var requestProducer = TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("edqs-request-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(edqsConfig.getRequestsTopic())) + .admin(edqsRequestsAdmin); + + var responseConsumer = TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(edqsConfig.getResponsesTopic() + "." + serviceInfoProvider.getServiceId())) + .clientId("monolith-edqs-response-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("monolith-edqs-response-consumer-" + serviceInfoProvider.getServiceId())) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), FromEdqsMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(edqsRequestsAdmin) + .statsService(consumerStatsService); + + return DefaultTbQueueRequestTemplate., TbProtoQueueMsg>builder() + .queueAdmin(edqsRequestsAdmin) + .requestTemplate(requestProducer.build()) + .responseTemplate(responseConsumer.build()) + .maxPendingRequests(edqsConfig.getMaxPendingRequests()) + .maxRequestTimeout(edqsConfig.getMaxRequestTimeout()) + .pollInterval(edqsConfig.getPollInterval()) + .build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -523,5 +652,9 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi if (edgeAdmin != null) { edgeAdmin.destroy(); } + if (cfAdmin != null) { + cfAdmin.destroy(); + } } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index c2f4649106..f3adb266a4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -24,11 +24,15 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -47,12 +51,15 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; @@ -79,6 +86,8 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; + private final TbQueueCalculatedFieldSettings calculatedFieldSettings; + private final EdqsConfig edqsConfig; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -93,6 +102,9 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin housekeeperReprocessingAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin cfAdmin; + private final TbQueueAdmin edqsEventsAdmin; + private final TbKafkaAdmin edqsRequestsAdmin; private final AtomicLong consumerCount = new AtomicLong(); private final AtomicLong edgeConsumerCount = new AtomicLong(); @@ -108,6 +120,8 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueEdgeSettings edgeSettings, TbKafkaConsumerStatsService consumerStatsService, TbQueueTransportNotificationSettings transportNotificationSettings, + TbQueueCalculatedFieldSettings calculatedFieldSettings, + EdqsConfig edqsConfig, TbKafkaTopicConfigs kafkaTopicConfigs) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; @@ -120,6 +134,8 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.consumerStatsService = consumerStatsService; this.transportNotificationSettings = transportNotificationSettings; this.edgeSettings = edgeSettings; + this.calculatedFieldSettings = calculatedFieldSettings; + this.edqsConfig = edqsConfig; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -134,6 +150,9 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); + this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsEventsConfigs()); + this.edqsRequestsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsRequestsConfigs()); } @Override @@ -440,6 +459,62 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createToCalculatedFieldMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-to-calculated-field-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(cfAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createToCalculatedFieldNotificationMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-calculated-field-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(notificationAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + return TbKafkaProducerTemplate.>builder() + .clientId("edqs-producer-" + queue.name().toLowerCase() + "-" + serviceInfoProvider.getServiceId()) + .settings(kafkaSettings) + .admin(edqsEventsAdmin) + .build(); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createEdqsRequestTemplate() { + var requestProducer = TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("edqs-request-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(edqsConfig.getRequestsTopic())) + .admin(edqsRequestsAdmin); + + var responseConsumer = TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(edqsConfig.getResponsesTopic() + "." + serviceInfoProvider.getServiceId())) + .clientId("tb-core-edqs-response-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("tb-core-edqs-response-consumer-" + serviceInfoProvider.getServiceId())) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), FromEdqsMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(edqsRequestsAdmin) + .statsService(consumerStatsService); + + return DefaultTbQueueRequestTemplate., TbProtoQueueMsg>builder() + .queueAdmin(edqsRequestsAdmin) + .requestTemplate(requestProducer.build()) + .responseTemplate(responseConsumer.build()) + .maxPendingRequests(edqsConfig.getMaxPendingRequests()) + .maxRequestTimeout(edqsConfig.getMaxRequestTimeout()) + .pollInterval(edqsConfig.getPollInterval()) + .build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -469,5 +544,9 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { if (vcAdmin != null) { vcAdmin.destroy(); } + if (cfAdmin != null) { + cfAdmin.destroy(); + } } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 1fc01c8df6..43fbb5efeb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -23,11 +23,16 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -43,12 +48,14 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; @@ -71,6 +78,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; + private final TbQueueCalculatedFieldSettings calculatedFieldSettings; private final TbQueueAdmin coreAdmin; private final TbKafkaAdmin ruleEngineAdmin; @@ -81,6 +89,9 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbQueueAdmin housekeeperAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin cfAdmin; + private final TbQueueAdmin cfStateAdmin; + private final TbQueueAdmin edqsEventsAdmin; private final AtomicLong consumerCount = new AtomicLong(); public KafkaTbRuleEngineQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -90,7 +101,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbKafkaConsumerStatsService consumerStatsService, TbQueueTransportNotificationSettings transportNotificationSettings, - TbQueueEdgeSettings edgeSettings, + TbQueueEdgeSettings edgeSettings, TbQueueCalculatedFieldSettings calculatedFieldSettings, TbKafkaTopicConfigs kafkaTopicConfigs) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; @@ -101,6 +112,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.consumerStatsService = consumerStatsService; this.transportNotificationSettings = transportNotificationSettings; this.edgeSettings = edgeSettings; + this.calculatedFieldSettings = calculatedFieldSettings; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -111,6 +123,9 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); + this.cfStateAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldStateConfigs()); + this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsEventsConfigs()); } @Override @@ -293,6 +308,81 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { .build(); } + @Override + public TbQueueConsumer> createToCalculatedFieldMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + consumerBuilder.clientId("tb-rule-engine-calculated-field-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.groupId(topicService.buildTopicName("tb-rule-engine-calculated-field-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(cfAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToCalculatedFieldMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-to-calculated-field-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(cfAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.getCalculatedFieldNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("tb-calculated-field-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId(topicService.buildTopicName("tb-calculated-field-notifications-node-") + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())) + .readFromBeginning(true) + .stopWhenRead(true) + .clientId("tb-rule-engine-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()) + .groupId(topicService.buildTopicName("tb-rule-engine-calculated-field-state-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), msg.getData() != null ? CalculatedFieldStateProto.parseFrom(msg.getData()) : null, msg.getHeaders())) + .admin(cfStateAdmin) + .statsService(consumerStatsService) + .build(); + } + + @Override + public TbQueueProducer> createCalculatedFieldStateProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("tb-rule-engine-to-calculated-field-state-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())) + .admin(cfStateAdmin) + .build(); + } + + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + return TbKafkaProducerTemplate.>builder() + .clientId("edqs-producer-" + queue.name().toLowerCase() + "-" + serviceInfoProvider.getServiceId()) + .settings(kafkaSettings) + .admin(edqsEventsAdmin) + .build(); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createEdqsRequestTemplate() { + throw new UnsupportedOperationException(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -313,5 +403,8 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { if (fwUpdatesAdmin != null) { fwUpdatesAdmin.destroy(); } + if (cfAdmin != null) { + cfAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index a89c901d73..037d1f2087 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -18,6 +18,8 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -42,7 +44,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; * Responsible for initialization of various Producers and Consumers used by TB Core Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory { +public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory, EdqsClientQueueFactory { /** * Used to push messages to instances of TB Transport Service @@ -159,4 +161,8 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, Hous return null; } + TbQueueProducer> createToCalculatedFieldMsgProducer(); + + TbQueueProducer> createToCalculatedFieldNotificationMsgProducer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index c65d12dfe6..7c3e415e9f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -17,6 +17,9 @@ package org.thingsboard.server.queue.provider; import jakarta.annotation.PostConstruct; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -48,6 +51,8 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toUsageStats; private TbQueueProducer> toVersionControl; private TbQueueProducer> toHousekeeper; + private TbQueueProducer> toCalculatedFields; + private TbQueueProducer> toCalculatedFieldNotifications; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -66,6 +71,8 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { this.toEdge = tbQueueProvider.createEdgeMsgProducer(); this.toEdgeNotifications = tbQueueProvider.createEdgeNotificationsMsgProducer(); this.toEdgeEvents = tbQueueProvider.createEdgeEventMsgProducer(); + this.toCalculatedFields = tbQueueProvider.createToCalculatedFieldMsgProducer(); + this.toCalculatedFieldNotifications = tbQueueProvider.createToCalculatedFieldNotificationMsgProducer(); } @Override @@ -124,4 +131,14 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { return toEdgeEvents; } + @Override + public TbQueueProducer> getCalculatedFieldsMsgProducer() { + return toCalculatedFields; + } + + @Override + public TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer() { + return toCalculatedFieldNotifications; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index 5bfb4675cd..865637b2ff 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.queue.provider; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -76,7 +78,7 @@ public interface TbQueueProducerProvider { */ TbQueueProducer> getTbUsageStatsMsgProducer(); - /** + /** * Used to push messages to other instances of TB Core Service * * @return @@ -91,4 +93,8 @@ public interface TbQueueProducerProvider { TbQueueProducer> getTbEdgeEventsMsgProducer(); + TbQueueProducer> getCalculatedFieldsMsgProducer(); + + TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index 8d1ec7f7f6..dcadf02d02 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -18,6 +18,8 @@ package org.thingsboard.server.queue.provider; import jakarta.annotation.PostConstruct; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -47,6 +49,7 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toEdge; private TbQueueProducer> toEdgeNotifications; private TbQueueProducer> toEdgeEvents; + private TbQueueProducer> toCalculatedFields; public TbRuleEngineProducerProvider(TbRuleEngineQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -64,6 +67,7 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { this.toEdge = tbQueueProvider.createEdgeMsgProducer(); this.toEdgeNotifications = tbQueueProvider.createEdgeNotificationsMsgProducer(); this.toEdgeEvents = tbQueueProvider.createEdgeEventMsgProducer(); + this.toCalculatedFields = tbQueueProvider.createToCalculatedFieldMsgProducer(); } @Override @@ -121,4 +125,14 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { return toHousekeeper; } + @Override + public TbQueueProducer> getCalculatedFieldsMsgProducer() { + return toCalculatedFields; + } + + @Override + public TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Rule Engine Service!"); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index 01523e0b87..767fea9f0c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -17,6 +17,9 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -36,7 +39,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; * Responsible for initialization of various Producers and Consumers used by TB Core Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory { +public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory, EdqsClientQueueFactory { /** * Used to push messages to instances of TB Transport Service @@ -109,11 +112,22 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory } /** - * Used to consume high priority messages by TB Core Service + * Used to consume high priority messages by TB Rule Engine Service * * @return */ TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer(); TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); + + TbQueueConsumer> createToCalculatedFieldMsgConsumer(); + + TbQueueProducer> createToCalculatedFieldMsgProducer(); + + TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer(); + + TbQueueConsumer> createCalculatedFieldStateConsumer(); + + TbQueueProducer> createCalculatedFieldStateProducer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index e201ddc357..a7a34992cd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import jakarta.annotation.PostConstruct; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -111,4 +112,13 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider return toHousekeeper; } + @Override + public TbQueueProducer> getCalculatedFieldsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } + + @Override + public TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java index d5201e6518..85c400d094 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import jakarta.annotation.PostConstruct; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -107,4 +108,14 @@ public class TbVersionControlProducerProvider implements TbQueueProducerProvider return toHousekeeper; } + @Override + public TbQueueProducer> getCalculatedFieldsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCalculatedFieldSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCalculatedFieldSettings.java new file mode 100644 index 0000000000..c2de8eff4e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCalculatedFieldSettings.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +@Lazy +@Data +@Component +public class TbQueueCalculatedFieldSettings { + + @Value("${queue.calculated_fields.event_topic}") + private String eventTopic; + + @Value("${queue.calculated_fields.state_topic}") + private String stateTopic; + + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java index 8bd58c4e81..46c29b867b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java @@ -38,6 +38,9 @@ public @interface AfterStartUp { int ACTOR_SYSTEM = 9; int REGULAR_SERVICE = 10; + int CF_READ_PROFILE_ENTITIES_SERVICE = 10; + int CF_READ_CF_SERVICE = 11; + int BEFORE_TRANSPORT_SERVICE = Integer.MAX_VALUE - 1001; int TRANSPORT_SERVICE = Integer.MAX_VALUE - 1000; int AFTER_TRANSPORT_SERVICE = Integer.MAX_VALUE - 999; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java index ca34d4006c..6030eb278d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java @@ -43,9 +43,8 @@ public class PropertyUtils { } public static Map getProps(Map defaultProperties, String propertiesStr, Function> parser) { - Map properties = defaultProperties; + Map properties = new HashMap<>(defaultProperties); if (StringUtils.isNotBlank(propertiesStr)) { - properties = new HashMap<>(properties); properties.putAll(parser.apply(propertiesStr)); } return properties; diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index 1760555e07..9e493220ca 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -37,23 +37,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.*; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.BDDMockito.willAnswer; -import static org.mockito.BDDMockito.willDoNothing; -import static org.mockito.BDDMockito.willReturn; -import static org.mockito.Mockito.RETURNS_DEEP_STUBS; -import static org.mockito.Mockito.atLeastOnce; -import static org.mockito.Mockito.lenient; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; +import static org.mockito.BDDMockito.*; +import static org.mockito.Mockito.*; import static org.mockito.hamcrest.MockitoHamcrest.longThat; @Slf4j @@ -145,19 +133,19 @@ public class DefaultTbQueueRequestTemplateTest { @Test public void givenMessages_whenSend_thenOK() { - willDoNothing().given(inst).sendToRequestTemplate(any(), any(), any(), any()); + willDoNothing().given(inst).sendToRequestTemplate(any(), any(), any(), any(), any()); inst.init(); final int msgCount = 10; for (int i = 0; i < msgCount; i++) { inst.send(getRequestMsgMock()); } assertThat(inst.pendingRequests.mappingCount(), equalTo((long) msgCount)); - verify(inst, times(msgCount)).sendToRequestTemplate(any(), any(), any(), any()); + verify(inst, times(msgCount)).sendToRequestTemplate(any(), any(), any(), any(), any()); } @Test public void givenMessagesOverMaxPendingRequests_whenSend_thenImmediateFailedFutureForTheOfRequests() { - willDoNothing().given(inst).sendToRequestTemplate(any(), any(), any(), any()); + willDoNothing().given(inst).sendToRequestTemplate(any(), any(), any(), any(), any()); inst.init(); int msgOverflowCount = 10; for (int i = 0; i < inst.maxPendingRequests; i++) { @@ -167,7 +155,7 @@ public class DefaultTbQueueRequestTemplateTest { assertThat("max pending requests overflow", inst.send(getRequestMsgMock()).isDone(), is(true)); //overflow, immediate failed future } assertThat(inst.pendingRequests.mappingCount(), equalTo(inst.maxPendingRequests)); - verify(inst, times((int) inst.maxPendingRequests)).sendToRequestTemplate(any(), any(), any(), any()); + verify(inst, times((int) inst.maxPendingRequests)).sendToRequestTemplate(any(), any(), any(), any(), any()); } @SuppressWarnings("unchecked") diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/AbstractScriptInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/AbstractScriptInvokeService.java index efe6fd7781..7cbac0401d 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/AbstractScriptInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/AbstractScriptInvokeService.java @@ -22,6 +22,8 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.script.api.tbel.TbelCfArg; +import org.thingsboard.script.api.tbel.TbelCfObject; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -256,6 +258,8 @@ public abstract class AbstractScriptInvokeService implements ScriptInvokeService for (Object arg : args) { if (arg instanceof CharSequence) { totalArgsSize += ((CharSequence) arg).length(); + } else if (arg instanceof TbelCfObject tbelCfObj) { + totalArgsSize += tbelCfObj.memorySize(); } else { var str = JacksonUtil.toString(arg); if (str != null) { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java index 1eeed4a5a9..c1d8ae929a 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java @@ -16,5 +16,5 @@ package org.thingsboard.script.api; public enum ScriptType { - RULE_NODE_SCRIPT + RULE_NODE_SCRIPT, CALCULATED_FIELD_SCRIPT } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java index c49e81052c..3680e1bfb2 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java @@ -130,9 +130,17 @@ public class DefaultTbelInvokeService extends AbstractScriptInvokeService implem OptimizerFactory.setDefaultOptimizer(OptimizerFactory.SAFE_REFLECTIVE); parserConfig = ParserContext.enableSandboxedMode(); parserConfig.addImport("JSON", TbJson.class); - parserConfig.registerDataType("Date", TbDate.class, date -> 8L); - parserConfig.registerDataType("Random", Random.class, date -> 8L); - parserConfig.registerDataType("Calendar", Calendar.class, date -> 8L); + parserConfig.registerDataType("Date", TbDate.class, val -> 8L); + parserConfig.registerDataType("Random", Random.class, val -> 8L); + parserConfig.registerDataType("Calendar", Calendar.class, val -> 8L); + parserConfig.registerDataType("TbelCfSingleValueArg", TbelCfSingleValueArg.class, TbelCfSingleValueArg::memorySize); + parserConfig.registerDataType("TbelCfTsRollingArg", TbelCfTsRollingArg.class, TbelCfTsRollingArg::memorySize); + parserConfig.registerDataType("TbelCfTsDoubleVal", TbelCfTsDoubleVal.class, TbelCfTsDoubleVal::memorySize); + parserConfig.registerDataType("TbelCfTsRollingData", TbelCfTsRollingData.class, TbelCfTsRollingData::memorySize); + parserConfig.registerDataType("TbTimeWindow", TbTimeWindow.class, TbTimeWindow::memorySize); + parserConfig.registerDataType("TbelCfTsDoubleVal", TbelCfTsMultiDoubleVal.class, TbelCfTsMultiDoubleVal::memorySize); + parserConfig.registerDataType("TbelCfCtx", TbelCfCtx.class, TbelCfCtx::memorySize); + TbUtils.register(parserConfig); executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, "tbel-executor")); try { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbDate.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbDate.java index 4528819e44..c674407a00 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbDate.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbDate.java @@ -37,6 +37,7 @@ import java.time.temporal.ChronoUnit; import java.time.temporal.TemporalAccessor; import java.util.Locale; import java.util.function.BiFunction; +import java.util.function.Function; public class TbDate implements Serializable, Cloneable { @@ -481,6 +482,43 @@ public class TbDate implements Serializable, Cloneable { instant = Instant.ofEpochMilli(dateMilliSecond); } + public void addDays(int days) { + adjustTime(zonedDateTime -> zonedDateTime.plusDays(days)); + } + + public void addYears(int years) { + adjustTime(zonedDateTime -> zonedDateTime.plusYears(years)); + } + + public void addMonths(int months) { + adjustTime(zonedDateTime -> zonedDateTime.plusMonths(months)); + } + + public void addWeeks(int weeks) { + adjustTime(zonedDateTime -> zonedDateTime.plusWeeks(weeks)); + } + + public void addHours(int hours) { + adjustTime(zonedDateTime -> zonedDateTime.plusHours(hours)); + } + + public void addMinutes(int minutes) { + adjustTime(zonedDateTime -> zonedDateTime.plusMinutes(minutes)); + } + + public void addSeconds(int seconds) { + adjustTime(zonedDateTime -> zonedDateTime.plusSeconds(seconds)); + } + + public void addNanos(long nanos) { + adjustTime(zonedDateTime -> zonedDateTime.plusNanos(nanos)); + } + + private void adjustTime(Function adjuster) { + ZonedDateTime zonedDateTime = adjuster.apply(getZonedDateTime()); + this.instant = zonedDateTime.toInstant(); + } + public ZoneOffset getLocaleZoneOffset(Instant... instants){ return ZoneId.systemDefault().getRules().getOffset(instants.length > 0 ? instants[0] : this.instant); } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java new file mode 100644 index 0000000000..1761ef6d0a --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbTimeWindow.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class TbTimeWindow implements TbelCfObject { + + public static final long OBJ_SIZE = 32L; + + private long startTs; + private long endTs; + + @Override + public long memorySize() { + return OBJ_SIZE; + } + + public boolean matches(long ts) { + return ts >= startTs && ts < endTs; + } +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 9a57240d23..ca3a8db02a 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -18,6 +18,8 @@ package org.thingsboard.script.api.tbel; import com.google.common.primitives.Bytes; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ArrayUtils; +import org.mvel2.ConversionHandler; +import org.mvel2.DataConversion; import org.mvel2.ExecutionContext; import org.mvel2.ParserConfiguration; import org.mvel2.execution.ExecutionArrayList; @@ -255,6 +257,8 @@ public class TbUtils { double.class, int.class))); parserConfig.addImport("toFixed", new MethodStub(TbUtils.class.getMethod("toFixed", float.class, int.class))); + parserConfig.addImport("toInt", new MethodStub(TbUtils.class.getMethod("toInt", + double.class))); parserConfig.addImport("hexToBytes", new MethodStub(TbUtils.class.getMethod("hexToBytes", ExecutionContext.class, String.class))); parserConfig.addImport("hexToBytesArray", new MethodStub(TbUtils.class.getMethod("hexToBytesArray", @@ -1155,6 +1159,10 @@ public class TbUtils { return BigDecimal.valueOf(value).setScale(precision, RoundingMode.HALF_UP).floatValue(); } + public static int toInt(double value) { + return BigDecimal.valueOf(value).setScale(0, RoundingMode.HALF_UP).intValue(); + } + public static ExecutionHashMap toFlatMap(ExecutionContext ctx, Map json) { return toFlatMap(ctx, json, new ArrayList<>(), true); } @@ -1299,7 +1307,7 @@ public class TbUtils { if (str == null || str.isEmpty()) { return -1; } - return str.matches("[+-]?\\d+(\\.\\d+)?") ? DEC_RADIX : -1; + return str.matches("[+-]?\\d+(\\.\\d+)?([eE][+-]?\\d+)?") ? DEC_RADIX : -1; } public static int isHexadecimal(String str) { @@ -1506,5 +1514,6 @@ public class TbUtils { } return hex; } + } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java new file mode 100644 index 0000000000..f95b08195e --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfArg.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = TbelCfSingleValueArg.class, name = "SINGLE_VALUE"), + @JsonSubTypes.Type(value = TbelCfTsRollingArg.class, name = "TS_ROLLING") +}) +public interface TbelCfArg extends TbelCfObject { + + @JsonIgnore + String getType(); + +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfCtx.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfCtx.java new file mode 100644 index 0000000000..ce42e2cf3b --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfCtx.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import lombok.Getter; + +import java.util.Collections; +import java.util.Map; + +public class TbelCfCtx implements TbelCfObject { + + @Getter + private final Map args; + + public TbelCfCtx(Map args) { + this.args = Collections.unmodifiableMap(args); + } + + @Override + public long memorySize() { + return OBJ_SIZE; + } +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfObject.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfObject.java new file mode 100644 index 0000000000..cf575bd9d0 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfObject.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +public interface TbelCfObject { + + long OBJ_SIZE = 32L; // Approximate calculation; + + long memorySize(); + +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfSingleValueArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfSingleValueArg.java new file mode 100644 index 0000000000..84227a8d80 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfSingleValueArg.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class TbelCfSingleValueArg implements TbelCfArg { + + private final long ts; + private final Object value; + + @JsonCreator + public TbelCfSingleValueArg( + @JsonProperty("ts") long ts, + @JsonProperty("value") Object value + ) { + this.ts = ts; + this.value = value; + } + + @Override + public long memorySize() { + if (value instanceof String strValue) { + return OBJ_SIZE + strValue.length(); + } else { + return OBJ_SIZE; + } + } + + @Override + public String getType() { + return "SINGLE_VALUE"; + } + +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsDoubleVal.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsDoubleVal.java new file mode 100644 index 0000000000..71565f3e1d --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsDoubleVal.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import lombok.Data; + +@Data +public class TbelCfTsDoubleVal implements TbelCfObject { + + public static final long OBJ_SIZE = 32L; // Approximate calculation; + + private final long ts; + private final double value; + + @Override + public long memorySize() { + return OBJ_SIZE; + } +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsMultiDoubleVal.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsMultiDoubleVal.java new file mode 100644 index 0000000000..2743bbd0de --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsMultiDoubleVal.java @@ -0,0 +1,66 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Data; + +@Data +public class TbelCfTsMultiDoubleVal implements TbelCfObject { + + public static final long OBJ_SIZE = 32L; // Approximate calculation; + + private final long ts; + private final double[] values; + + @JsonIgnore + public double getV1() { + return getV(0); + } + + @JsonIgnore + public double getV2() { + return getV(1); + } + + @JsonIgnore + public double getV3() { + return getV(2); + } + + @JsonIgnore + public double getV4() { + return getV(3); + } + + @JsonIgnore + public double getV5() { + return getV(4); + } + + private double getV(int idx) { + if (values.length < idx + 1) { + throw new IllegalArgumentException("Can't get value at index " + idx + ". There are " + values.length + " values present."); + } else { + return values[idx]; + } + } + + @Override + public long memorySize() { + return OBJ_SIZE + values.length * 8L; + } +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java new file mode 100644 index 0000000000..3b5a7ac9bd --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArg.java @@ -0,0 +1,368 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; +import org.thingsboard.common.util.JacksonUtil; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; +import java.util.function.Consumer; + +import static org.thingsboard.script.api.tbel.TbelCfTsDoubleVal.OBJ_SIZE; + +public class TbelCfTsRollingArg implements TbelCfArg, Iterable { + + @Getter + private final TbTimeWindow timeWindow; + @Getter + private final List values; + + @JsonCreator + public TbelCfTsRollingArg( + @JsonProperty("timeWindow") TbTimeWindow timeWindow, + @JsonProperty("values") List values + ) { + this.timeWindow = timeWindow; + this.values = Collections.unmodifiableList(values); + } + + public TbelCfTsRollingArg(long timeWindow, List values) { + long ts = System.currentTimeMillis(); + this.timeWindow = new TbTimeWindow(ts - timeWindow, ts); + this.values = Collections.unmodifiableList(values); + } + + @Override + public long memorySize() { + return 12 + values.size() * OBJ_SIZE; + } + + @JsonIgnore + public List getValue() { + return values; + } + + public double max() { + return max(true); + } + + public double max(boolean ignoreNaN) { + if (values.isEmpty()) { + throw new IllegalArgumentException("Rolling argument values are empty."); + } + + double max = Double.MIN_VALUE; + for (TbelCfTsDoubleVal value : values) { + double val = value.getValue(); + if (!ignoreNaN && Double.isNaN(val)) { + return val; + } + if (max < val) { + max = val; + } + } + return max; + } + + public double min() { + return min(true); + } + + public double min(boolean ignoreNaN) { + if (values.isEmpty()) { + throw new IllegalArgumentException("Rolling argument values are empty."); + } + + double min = Double.MAX_VALUE; + for (TbelCfTsDoubleVal value : values) { + double val = value.getValue(); + if (!ignoreNaN && Double.isNaN(val)) { + return Double.NaN; + } + if (min > val) { + min = val; + } + } + return min; + } + + public double avg() { + return avg(true); + } + + public double avg(boolean ignoreNaN) { + return mean(ignoreNaN); + } + + public double mean() { + return mean(true); + } + + public double mean(boolean ignoreNaN) { + if (values.isEmpty()) { + throw new IllegalArgumentException("Rolling argument values are empty."); + } + + return sum(ignoreNaN) / count(ignoreNaN); + } + + public double std() { + return std(true); + } + + public double std(boolean ignoreNaN) { + if (values.isEmpty()) { + throw new IllegalArgumentException("Rolling argument values are empty."); + } + + double mean = mean(ignoreNaN); + if (!ignoreNaN && Double.isNaN(mean)) { + return Double.NaN; + } + + double sum = 0; + for (TbelCfTsDoubleVal value : values) { + double val = value.getValue(); + if (Double.isNaN(val)) { + if (!ignoreNaN) { + return Double.NaN; + } + } else { + sum += Math.pow(val - mean, 2); + } + } + return Math.sqrt(sum / count(ignoreNaN)); + } + + public double median() { + return median(true); + } + + public double median(boolean ignoreNaN) { + if (values.isEmpty()) { + throw new IllegalArgumentException("Rolling argument values are empty."); + } + + List sortedValues = new ArrayList<>(); + for (TbelCfTsDoubleVal value : values) { + double val = value.getValue(); + if (Double.isNaN(val)) { + if (!ignoreNaN) { + return Double.NaN; + } + } else { + sortedValues.add(val); + } + } + Collections.sort(sortedValues); + + int size = sortedValues.size(); + return (size % 2 == 1) + ? sortedValues.get(size / 2) + : (sortedValues.get(size / 2 - 1) + sortedValues.get(size / 2)) / 2.0; + } + + public int count() { + return count(true); + } + + public int count(boolean ignoreNaN) { + int count = 0; + if (ignoreNaN) { + for (TbelCfTsDoubleVal value : values) { + if (!Double.isNaN(value.getValue())) { + count++; + } + } + return count; + } + return values.size(); + } + + public double last() { + return last(true); + } + + public double last(boolean ignoreNaN) { + if (values.isEmpty()) { + throw new IllegalArgumentException("Rolling argument values are empty."); + } + + double value = values.get(values.size() - 1).getValue(); + if (!Double.isNaN(value) || !ignoreNaN) { + return value; + } + for (int i = values.size() - 2; i >= 0; i--) { + double prevValue = values.get(i).getValue(); + if (!Double.isNaN(prevValue)) { + return prevValue; + } + } + throw new IllegalArgumentException("Rolling argument values are empty."); + } + + public double first() { + return first(true); + } + + public double first(boolean ignoreNaN) { + if (values.isEmpty()) { + throw new IllegalArgumentException("Rolling argument values are empty."); + } + + double firstValue = values.get(0).getValue(); + if (!Double.isNaN(firstValue) || !ignoreNaN) { + return firstValue; + } + for (int i = 1; i < values.size(); i++) { + double nextValue = values.get(i).getValue(); + if (!Double.isNaN(nextValue)) { + return nextValue; + } + } + throw new IllegalArgumentException("Rolling argument values are empty."); + } + + public double sum() { + return sum(true); + } + + public double sum(boolean ignoreNaN) { + if (values.isEmpty()) { + throw new IllegalArgumentException("Rolling argument values are empty."); + } + + double sum = 0; + for (TbelCfTsDoubleVal value : values) { + double val = value.getValue(); + if (Double.isNaN(val)) { + if (!ignoreNaN) { + return Double.NaN; + } + } else { + sum += val; + } + } + return sum; + } + + public TbelCfTsRollingData merge(TbelCfTsRollingArg other) { + return mergeAll(Collections.singletonList(other), null); + } + + public TbelCfTsRollingData merge(TbelCfTsRollingArg other, Map settings) { + return mergeAll(Collections.singletonList(other), settings); + } + + public TbelCfTsRollingData mergeAll(List others) { + return mergeAll(others, null); + } + + public TbelCfTsRollingData mergeAll(List others, Map settings) { + List args = new ArrayList<>(others.size() + 1); + args.add(this); + args.addAll(others); + + boolean ignoreNaN = true; + if (settings != null && settings.containsKey("ignoreNaN")) { + ignoreNaN = Boolean.parseBoolean(settings.get("ignoreNaN").toString()); + } + + TbTimeWindow timeWindow = null; + if (settings != null && settings.containsKey("timeWindow")) { + var twVar = settings.get("timeWindow"); + if (twVar instanceof TbTimeWindow) { + timeWindow = (TbTimeWindow) settings.get("timeWindow"); + } else if (twVar instanceof Map twMap) { + timeWindow = new TbTimeWindow(Long.valueOf(twMap.get("startTs").toString()), Long.valueOf(twMap.get("endTs").toString())); + } else { + timeWindow = JacksonUtil.fromString(settings.get("timeWindow").toString(), TbTimeWindow.class); + } + } + + TreeSet allTimestamps = new TreeSet<>(); + long startTs = Long.MAX_VALUE; + long endTs = Long.MIN_VALUE; + for (TbelCfTsRollingArg arg : args) { + for (TbelCfTsDoubleVal val : arg.getValues()) { + allTimestamps.add(val.getTs()); + } + startTs = Math.min(startTs, arg.getTimeWindow().getStartTs()); + endTs = Math.max(endTs, arg.getTimeWindow().getEndTs()); + } + + List data = new ArrayList<>(); + + int[] lastIndex = new int[args.size()]; + double[] result = new double[args.size()]; + Arrays.fill(result, Double.NaN); + + for (long ts : allTimestamps) { + for (int i = 0; i < args.size(); i++) { + var arg = args.get(i); + var values = arg.getValues(); + while (lastIndex[i] < values.size() && values.get(lastIndex[i]).getTs() <= ts) { + result[i] = values.get(lastIndex[i]).getValue(); + lastIndex[i]++; + } + } + if (timeWindow == null || timeWindow.matches(ts)) { + if (ignoreNaN) { + boolean skip = false; + for (int i = 0; i < args.size(); i++) { + if (Double.isNaN(result[i])) { + skip = true; + break; + } + } + if (!skip) { + data.add(new TbelCfTsMultiDoubleVal(ts, Arrays.copyOf(result, result.length))); + } + } else { + data.add(new TbelCfTsMultiDoubleVal(ts, Arrays.copyOf(result, result.length))); + } + } + } + + return new TbelCfTsRollingData(timeWindow != null ? timeWindow : new TbTimeWindow(startTs, endTs), data); + } + + + @JsonIgnore + public int getSize() { + return values.size(); + } + + @Override + public Iterator iterator() { + return values.iterator(); + } + + @Override + public String getType() { + return "TS_ROLLING"; + } + +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingData.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingData.java new file mode 100644 index 0000000000..646e826915 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbelCfTsRollingData.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.function.Consumer; + +import static org.thingsboard.script.api.tbel.TbelCfTsDoubleVal.OBJ_SIZE; + +public class TbelCfTsRollingData implements TbelCfObject, Iterable { + + @Getter + private final TbTimeWindow timeWindow; + @Getter + private final List values; + + public TbelCfTsRollingData(TbTimeWindow timeWindow, List values) { + this.timeWindow = timeWindow; + this.values = Collections.unmodifiableList(values); + } + + @Override + public long memorySize() { + return 12 + values.size() * OBJ_SIZE; + } + + @JsonIgnore + public List getValue() { + return values; + } + + @JsonIgnore + public int getSize() { + return values.size(); + } + + @Override + public Iterator iterator() { + return values.iterator(); + } + +} diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbDateTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbDateTest.java index 9492c2e610..e001e9fb11 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbDateTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbDateTest.java @@ -885,4 +885,64 @@ class TbDateTest { Assertions.assertNotNull(serializedTbDate); Assertions.assertEquals(expectedDate.toString(), serializedTbDate); } + + @Test + void testAddFunctions() { + TbDate d = new TbDate(2024, 1, 1, 10, 0, 0, 0); + testResultChangeDateTime(d); + + d.addYears(1); + testResultChangeDateTime(d); + + d.addYears(-2); + testResultChangeDateTime(d); + + d.addMonths(2); + testResultChangeDateTime(d); + + d.addMonths(10); + testResultChangeDateTime(d); + + d.addMonths(-13); + testResultChangeDateTime(d); + + d.addWeeks(4); + testResultChangeDateTime(d); + + d.addWeeks(-5); + testResultChangeDateTime(d); + + d.addDays(6); + testResultChangeDateTime(d); + + d.addDays(45); + testResultChangeDateTime(d); + + d.addDays(-50); + testResultChangeDateTime(d); + + d.addHours(23); + testResultChangeDateTime(d); + + d.addHours(-47); + testResultChangeDateTime(d); + + d.addMinutes(59); + testResultChangeDateTime(d); + + d.addMinutes(-60); + testResultChangeDateTime(d); + + d.addSeconds(59); + testResultChangeDateTime(d); + + d.addSeconds(-60); + testResultChangeDateTime(d); + + d.addNanos(999999); + testResultChangeDateTime(d); + + d.addNanos(-1000000); + testResultChangeDateTime(d); + } } diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index 72d70e1d92..69987d3132 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -442,8 +442,9 @@ public class TbUtilsTest { @Test public void parsDouble() { - String doubleValStr = "1729.1729"; - Assertions.assertEquals(java.util.Optional.of(doubleVal).get(), TbUtils.parseDouble(doubleValStr)); + String doubleValStr = "1.1428250947E8"; + Assertions.assertEquals(Double.parseDouble(doubleValStr), TbUtils.parseDouble(doubleValStr)); + doubleValStr = "1729.1729"; Assertions.assertEquals(0, Double.compare(doubleVal, TbUtils.parseHexToDouble(longValHex))); Assertions.assertEquals(0, Double.compare(doubleValRev, TbUtils.parseHexToDouble(longValHex, false))); Assertions.assertEquals(0, Double.compare(doubleVal, TbUtils.parseBigEndianHexToDouble(longValHex))); @@ -930,7 +931,13 @@ public class TbUtilsTest { @Test public void isDecimal_Test() { Assertions.assertEquals(10, TbUtils.isDecimal("4567039")); + Assertions.assertEquals(10, TbUtils.isDecimal("1.1428250947E8")); + Assertions.assertEquals(10, TbUtils.isDecimal("123.45")); + Assertions.assertEquals(10, TbUtils.isDecimal("-1.23E-4")); + Assertions.assertEquals(10, TbUtils.isDecimal("1E5")); Assertions.assertEquals(-1, TbUtils.isDecimal("C100110")); + Assertions.assertEquals(-1, TbUtils.isDecimal("abc")); + Assertions.assertEquals(-1, TbUtils.isDecimal(null)); } @Test @@ -1102,7 +1109,7 @@ public class TbUtilsTest { String validInput = Base64.getEncoder().encodeToString(new byte[]{1, 2, 3, 4, 5}); ExecutionArrayList actual = TbUtils.base64ToBytesList(ctx, validInput); ExecutionArrayList expected = new ExecutionArrayList<>(ctx); - expected.addAll(List.of((byte) 1, (byte)2, (byte)3, (byte)4, (byte)5)); + expected.addAll(List.of((byte) 1, (byte) 2, (byte) 3, (byte) 4, (byte) 5)); Assertions.assertEquals(expected, actual); String emptyInput = Base64.getEncoder().encodeToString(new byte[]{}); @@ -1116,6 +1123,7 @@ public class TbUtilsTest { TbUtils.base64ToBytesList(ctx, null); }); } + @Test public void bytesToHex_Test() { byte[] bb = {(byte) 0xBB, (byte) 0xAA}; @@ -1129,6 +1137,13 @@ public class TbUtilsTest { Assertions.assertEquals(expected, actual); } + @Test + void toInt() { + Assertions.assertEquals(1729, TbUtils.toInt(doubleVal)); + Assertions.assertEquals(13, TbUtils.toInt(12.8)); + Assertions.assertEquals(28, TbUtils.toInt(28.0)); + } + private static List toList(byte[] data) { List result = new ArrayList<>(data.length); for (Byte b : data) { diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java new file mode 100644 index 0000000000..69eba2fab2 --- /dev/null +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbelCfTsRollingArgTest.java @@ -0,0 +1,213 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.thingsboard.common.util.JacksonUtil; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.within; + +public class TbelCfTsRollingArgTest { + + private final long ts = System.currentTimeMillis(); + + private TbelCfTsRollingArg rollingArg; + + @BeforeEach + void setUp() { + rollingArg = new TbelCfTsRollingArg( + new TbTimeWindow(ts - 30000, ts - 10), + List.of( + new TbelCfTsDoubleVal(ts - 10, Double.NaN), + new TbelCfTsDoubleVal(ts - 20, 2.0), + new TbelCfTsDoubleVal(ts - 30, 8.0), + new TbelCfTsDoubleVal(ts - 40, Double.NaN), + new TbelCfTsDoubleVal(ts - 50, 3.0), + new TbelCfTsDoubleVal(ts - 60, 9.0), + new TbelCfTsDoubleVal(ts - 70, Double.NaN) + ) + ); + } + + @Test + void testMax() { + assertThat(rollingArg.max()).isEqualTo(9.0); + assertThat(rollingArg.max(false)).isNaN(); + } + + @Test + void testMin() { + assertThat(rollingArg.min()).isEqualTo(2.0); + assertThat(rollingArg.min(false)).isNaN(); + } + + @Test + void testMean() { + assertThat(rollingArg.mean()).isEqualTo(5.5); + assertThat(rollingArg.mean(false)).isNaN(); + } + + @Test + void testStd() { + assertThat(rollingArg.std()).isCloseTo(3.0413812651491097, within(0.001)); + assertThat(rollingArg.std(false)).isNaN(); + } + + @Test + void testMedian() { + assertThat(rollingArg.median()).isEqualTo(5.5); + assertThat(rollingArg.median(false)).isNaN(); + } + + @Test + void testCount() { + assertThat(rollingArg.count()).isEqualTo(4); + assertThat(rollingArg.count(false)).isEqualTo(7); + } + + @Test + void testLast() { + assertThat(rollingArg.last()).isEqualTo(9.0); + assertThat(rollingArg.last(false)).isNaN(); + } + + @Test + void testFirst() { + assertThat(rollingArg.first()).isEqualTo(2.0); + assertThat(rollingArg.first(false)).isNaN(); + } + + @Test + void testFirstAndLastWhenOnlyNaNAndIgnoreNaNIsFalse() { + assertThat(rollingArg.first()).isEqualTo(2.0); + rollingArg = new TbelCfTsRollingArg( + new TbTimeWindow(ts - 30000, ts - 10), + List.of( + new TbelCfTsDoubleVal(ts - 10, Double.NaN), + new TbelCfTsDoubleVal(ts - 40, Double.NaN), + new TbelCfTsDoubleVal(ts - 70, Double.NaN) + ) + ); + assertThatThrownBy(rollingArg::first).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::last).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + } + + @Test + void testSum() { + assertThat(rollingArg.sum()).isEqualTo(22.0); + assertThat(rollingArg.sum(false)).isNaN(); + } + + @Test + void testEmptyValues() { + rollingArg = new TbelCfTsRollingArg(new TbTimeWindow(0, 10), List.of()); + assertThatThrownBy(rollingArg::sum).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::max).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::min).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::mean).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::std).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::median).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::first).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + assertThatThrownBy(rollingArg::last).isInstanceOf(IllegalArgumentException.class).hasMessage("Rolling argument values are empty."); + } + + @Test + public void merge_two_rolling_args_ts_match_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = arg1.merge(arg2); + Assertions.assertEquals(3, result.getSize()); + Assertions.assertNotNull(result.getValues()); + Assertions.assertNotNull(result.getValues().get(0)); + Assertions.assertEquals(1000L, result.getValues().get(0).getTs()); + Assertions.assertEquals(1, result.getValues().get(0).getValues()[0]); + Assertions.assertEquals(11, result.getValues().get(0).getValues()[1]); + } + + @Test + public void merge_two_rolling_args_with_timewindow_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(1000, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = arg1.merge(arg2, Collections.singletonMap("timeWindow", new TbTimeWindow(0, 10000))); + Assertions.assertEquals(2, result.getSize()); + Assertions.assertNotNull(result.getValues()); + Assertions.assertNotNull(result.getValues().get(0)); + Assertions.assertEquals(1000L, result.getValues().get(0).getTs()); + Assertions.assertEquals(1, result.getValues().get(0).getValues()[0]); + Assertions.assertEquals(11, result.getValues().get(0).getValues()[1]); + + result = arg1.merge(arg2, Collections.singletonMap("timeWindow", Map.of("startTs", 0L, "endTs", 10000))); + Assertions.assertEquals(2, result.getSize()); + Assertions.assertNotNull(result.getValues()); + Assertions.assertNotNull(result.getValues().get(0)); + Assertions.assertEquals(1000L, result.getValues().get(0).getTs()); + Assertions.assertEquals(1, result.getValues().get(0).getValues()[0]); + Assertions.assertEquals(11, result.getValues().get(0).getValues()[1]); + } + + @Test + public void merge_two_rolling_args_ts_mismatch_default_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(100, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(200, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = arg1.merge(arg2); + Assertions.assertEquals(3, result.getSize()); + Assertions.assertNotNull(result.getValues()); + + TbelCfTsMultiDoubleVal item0 = result.getValues().get(0); + Assertions.assertNotNull(item0); + Assertions.assertEquals(200L, item0.getTs()); + Assertions.assertEquals(1, item0.getValues()[0]); + Assertions.assertEquals(11, item0.getValues()[1]); + } + + @Test + public void merge_two_rolling_args_ts_mismatch_ignore_nan_disabled_test() { + TbTimeWindow tw = new TbTimeWindow(0, 60000); + TbelCfTsRollingArg arg1 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(100, 1), new TbelCfTsDoubleVal(5000, 2), new TbelCfTsDoubleVal(15000, 3))); + TbelCfTsRollingArg arg2 = new TbelCfTsRollingArg(tw, Arrays.asList(new TbelCfTsDoubleVal(200, 11), new TbelCfTsDoubleVal(5000, 12), new TbelCfTsDoubleVal(15000, 13))); + + var result = arg1.merge(arg2, Collections.singletonMap("ignoreNaN", false)); + Assertions.assertEquals(4, result.getSize()); + Assertions.assertNotNull(result.getValues()); + + TbelCfTsMultiDoubleVal item0 = result.getValues().get(0); + Assertions.assertNotNull(item0); + Assertions.assertEquals(100L, item0.getTs()); + Assertions.assertEquals(1, item0.getValues()[0]); + Assertions.assertEquals(Double.NaN, item0.getValues()[1]); + + TbelCfTsMultiDoubleVal item1 = result.getValues().get(1); + Assertions.assertEquals(200L, item1.getTs()); + Assertions.assertEquals(1, item1.getValues()[0]); + Assertions.assertEquals(11, item1.getValues()[1]); + } + +} \ No newline at end of file diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java index a21b52eaf4..de64d4fe54 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java @@ -22,7 +22,8 @@ public enum StatsType { JS_INVOKE("jsInvoke"), RATE_EXECUTOR("rateExecutor"), HOUSEKEEPER("housekeeper"), - EDGE("edge"); + EDGE("edge"), + EDQS("edqs"); private final String name; diff --git a/common/util/src/main/java/org/thingsboard/common/util/DebugModeUtil.java b/common/util/src/main/java/org/thingsboard/common/util/DebugModeUtil.java index d50ecfc5e0..c4b062a0d8 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/DebugModeUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/DebugModeUtil.java @@ -57,4 +57,14 @@ public final class DebugModeUtil { return debugSettings != null && nodeConnections != null && debugSettings.isFailuresEnabled() && nodeConnections.contains(TbNodeConnectionType.FAILURE); } } + + public static boolean isDebugFailuresAvailable(HasDebugSettings debugSettingsAware) { + if (isDebugAllAvailable(debugSettingsAware)) { + return true; + } else { + var debugSettings = debugSettingsAware.getDebugSettings(); + return debugSettings != null && debugSettings.isFailuresEnabled(); + } + } + } diff --git a/common/util/src/main/java/org/thingsboard/common/util/DonAsynchron.java b/common/util/src/main/java/org/thingsboard/common/util/DonAsynchron.java index 008158246e..0f1a56cb17 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/DonAsynchron.java +++ b/common/util/src/main/java/org/thingsboard/common/util/DonAsynchron.java @@ -36,6 +36,9 @@ public class DonAsynchron { FutureCallback callback = new FutureCallback() { @Override public void onSuccess(T result) { + if (onSuccess == null) { + return; + } try { onSuccess.accept(result); } catch (Throwable th) { @@ -45,6 +48,9 @@ public class DonAsynchron { @Override public void onFailure(Throwable t) { + if (onFailure == null) { + return; + } onFailure.accept(t); } }; diff --git a/common/util/src/main/java/org/thingsboard/common/util/ExceptionUtil.java b/common/util/src/main/java/org/thingsboard/common/util/ExceptionUtil.java index e08a8de30a..c93ec09bbf 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/ExceptionUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/ExceptionUtil.java @@ -64,4 +64,14 @@ public class ExceptionUtil { } } } + + public static String getMessage(Throwable t) { + String message = t.getMessage(); + if (StringUtils.isNotEmpty(message)) { + return message; + } else { + return t.getClass().getSimpleName(); + } + } + } diff --git a/common/util/src/main/java/org/thingsboard/common/util/NoOpFutureCallback.java b/common/util/src/main/java/org/thingsboard/common/util/NoOpFutureCallback.java new file mode 100644 index 0000000000..176d949c95 --- /dev/null +++ b/common/util/src/main/java/org/thingsboard/common/util/NoOpFutureCallback.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.common.util; + +import com.google.common.util.concurrent.FutureCallback; + +public enum NoOpFutureCallback implements FutureCallback { + + INSTANCE; + + @Override + public void onSuccess(Object result) {} + + @Override + public void onFailure(Throwable t) {} + + @SuppressWarnings("unchecked") + public static FutureCallback instance() { + return (FutureCallback) INSTANCE; + } + +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index 737de124c8..0f8969f9a4 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -175,7 +175,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe } } } - consumer.subscribe(event.getPartitionsMap().values().stream().findAny().orElse(Collections.emptySet())); + consumer.subscribe(event.getNewPartitions().values().stream().findAny().orElse(Collections.emptySet())); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/Dao.java b/dao/src/main/java/org/thingsboard/server/dao/Dao.java index a96e401869..72883c55ef 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/Dao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/Dao.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; import org.thingsboard.server.common.data.id.TenantId; import java.util.Collection; @@ -45,6 +46,12 @@ public interface Dao { List findIdsByTenantIdAndIdOffset(TenantId tenantId, UUID idOffset, int limit); - default EntityType getEntityType() { return null; } + default List findNextBatch(UUID id, int batchSize) { + throw new UnsupportedOperationException(); + } + + default EntityType getEntityType() { + return null; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java index c9dcc87fa9..43b651bb24 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.dao.model.ToData; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -85,6 +86,10 @@ public abstract class DaoUtil { return toPageable(pageLink, Collections.emptyMap(), sortOrders); } + public static Pageable toPageable(PageLink pageLink, String... sortColumns) { + return toPageable(pageLink, Collections.emptyMap(), Arrays.stream(sortColumns).map(column -> new SortOrder(column, SortOrder.Direction.ASC)).toList(), false); + } + public static Pageable toPageable(PageLink pageLink, Map columnMap, List sortOrders) { return toPageable(pageLink, columnMap, sortOrders, true); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ResourceContainerDao.java b/dao/src/main/java/org/thingsboard/server/dao/ResourceContainerDao.java new file mode 100644 index 0000000000..6a952fd501 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/ResourceContainerDao.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.List; + +public interface ResourceContainerDao> { + + List findByTenantIdAndResourceLink(TenantId tenantId, String link, int limit); + + List findByResourceLink(String link, int limit); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java index 1ebf148b1c..c295108a47 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java @@ -16,8 +16,17 @@ package org.thingsboard.server.dao; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; -public interface TenantEntityDao { +public interface TenantEntityDao { + + default Long countByTenantId(TenantId tenantId) { + throw new UnsupportedOperationException(); + } + + default PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + throw new UnsupportedOperationException(); + } - Long countByTenantId(TenantId tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java index b0439b6407..0351490f70 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java @@ -106,7 +106,7 @@ public interface AlarmDao extends Dao { AlarmApiCallResult unassignAlarm(TenantId tenantId, AlarmId alarmId, long unassignTime); - long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query); + long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query, Collection orderedEntityIds); PageData findTenantAlarmTypes(UUID tenantId, PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index fcfd5e0a8d..23e05e1bfa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -351,8 +351,13 @@ public class BaseAlarmService extends AbstractCachedEntityService orderedEntityIds) { validateId(tenantId, id -> INCORRECT_TENANT_ID + id); - return alarmDao.countAlarmsByQuery(tenantId, customerId, query); + return alarmDao.countAlarmsByQuery(tenantId, customerId, query, orderedEntityIds); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index 3ac2877fc0..36700ff59f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.asset; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.id.AssetId; @@ -36,7 +37,7 @@ import java.util.UUID; * The Interface AssetDao. * */ -public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find asset info by id. @@ -103,6 +104,16 @@ public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityD */ PageData findAssetInfosByTenantIdAndAssetProfileId(UUID tenantId, UUID assetProfileId, PageLink pageLink); + /** + * Find asset ids by tenantId, assetProfileId and page link. + * + * @param tenantId the tenantId + * @param assetProfileId the assetProfileId + * @param pageLink the page link + * @return the list of asset objects + */ + PageData findAssetIdsByTenantIdAndAssetProfileId(UUID tenantId, UUID assetProfileId, PageLink pageLink); + /** * Find assets by tenantId and assets Ids. * @@ -226,4 +237,7 @@ public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityD PageData findAssetsByTenantIdAndEdgeIdAndType(UUID tenantId, UUID edgeId, String type, PageLink pageLink); PageData> getAllAssetTypes(PageLink pageLink); + + PageData findProfileEntityIdInfos(PageLink pageLink); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java index 3ba9160975..6bf44e4da1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java @@ -148,12 +148,11 @@ public class AssetProfileServiceImpl extends CachedVersionedEntityService findProfileEntityIdInfos(PageLink pageLink) { + log.trace("Executing findProfileEntityIdInfos, pageLink [{}]", pageLink); + validatePageLink(pageLink); + return assetDao.findProfileEntityIdInfos(pageLink); + } + + @Override + public PageData findAssetIdsByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink) { + log.trace("Executing findAssetIdsByTenantIdAndAssetProfileId, tenantId [{}], assetProfileId [{}]", tenantId, assetProfileId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(assetProfileId, id -> INCORRECT_ASSET_PROFILE_ID + id); + validatePageLink(pageLink); + return assetDao.findAssetIdsByTenantIdAndAssetProfileId(tenantId.getId(), assetProfileId.getId(), pageLink); + } + @Override public ListenableFuture> findAssetsByTenantIdAndIdsAsync(TenantId tenantId, List assetIds) { log.trace("Executing findAssetsByTenantIdAndIdsAsync, tenantId [{}], assetIds [{}]", tenantId, assetIds); diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java index 762201f58d..5527d17add 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java @@ -23,10 +23,12 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.dao.model.sql.AttributeKvEntity; import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.UUID; /** * @author Andrew Shvayka @@ -45,6 +47,8 @@ public interface AttributesDao { List>> removeAllWithVersions(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, List keys); + List findNextBatch(UUID entityId, int attributeType, int attributeKey, int batchSize); + List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId); List findAllKeysByEntityIds(TenantId tenantId, List entityIds); diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index 6e3759a991..777a77d054 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -17,6 +17,8 @@ package org.thingsboard.server.dao.attributes; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Value; @@ -24,13 +26,18 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.AttributeKv; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.dao.service.Validator; +import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Optional; @@ -45,16 +52,15 @@ import static org.thingsboard.server.dao.attributes.AttributeUtils.validate; @ConditionalOnProperty(prefix = "cache.attributes", value = "enabled", havingValue = "false", matchIfMissing = true) @Primary @Slf4j +@RequiredArgsConstructor public class BaseAttributesService implements AttributesService { + private final AttributesDao attributesDao; + private final EdqsService edqsService; @Value("${sql.attributes.value_no_xss_validation:false}") private boolean valueNoXssValidation; - public BaseAttributesService(AttributesDao attributesDao) { - this.attributesDao = attributesDao; - } - @Override public ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, String attributeKey) { validate(entityId, scope); @@ -98,26 +104,53 @@ public class BaseAttributesService implements AttributesService { public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { validate(entityId, scope); AttributeUtils.validate(attribute, valueNoXssValidation); - return attributesDao.save(tenantId, entityId, scope, attribute); + return doSave(tenantId, entityId, scope, attribute); } @Override public ListenableFuture> save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { validate(entityId, scope); AttributeUtils.validate(attributes, valueNoXssValidation); - List> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, scope, attribute)).collect(Collectors.toList()); + List> saveFutures = attributes.stream().map(attribute -> doSave(tenantId, entityId, scope, attribute)).collect(Collectors.toList()); return Futures.allAsList(saveFutures); } + private ListenableFuture doSave(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { + ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); + return Futures.transform(future, version -> { + edqsService.onUpdate(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, attribute, version)); + return version; + }, MoreExecutors.directExecutor()); + } + @Override public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributeKeys) { validate(entityId, scope); - return Futures.allAsList(attributesDao.removeAll(tenantId, entityId, scope, attributeKeys)); + List>> futures = attributesDao.removeAllWithVersions(tenantId, entityId, scope, attributeKeys); + return Futures.transform(Futures.allAsList(futures), result -> { + List keys = new ArrayList<>(); + for (TbPair keyVersionPair : result) { + String key = keyVersionPair.getFirst(); + Long version = keyVersionPair.getSecond(); + if (version != null) { + edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, version)); + } + keys.add(key); + } + return keys; + }, MoreExecutors.directExecutor()); } @Override public int removeAllByEntityId(TenantId tenantId, EntityId entityId) { List> deleted = attributesDao.removeAllByEntityId(tenantId, entityId); + deleted.forEach(attribute -> { + AttributeScope scope = attribute.getKey(); + String key = attribute.getValue(); + if (scope != null && key != null) { + edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, Long.MAX_VALUE)); + } + }); return deleted.size(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index d9afa69ba7..559828911f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -24,18 +24,22 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.TbCacheValueWrapper; import org.thingsboard.server.cache.VersionedTbCache; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.AttributeKv; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.common.stats.DefaultCounter; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.cache.CacheExecutorService; @@ -67,6 +71,7 @@ public class CachedAttributesService implements AttributesService { private final AttributesDao attributesDao; private final JpaExecutorService jpaExecutorService; private final CacheExecutorService cacheExecutorService; + private final EdqsService edqsService; private final DefaultCounter hitCounter; private final DefaultCounter missCounter; private final VersionedTbCache cache; @@ -79,11 +84,12 @@ public class CachedAttributesService implements AttributesService { public CachedAttributesService(AttributesDao attributesDao, JpaExecutorService jpaExecutorService, - StatsFactory statsFactory, + @Lazy EdqsService edqsService, StatsFactory statsFactory, CacheExecutorService cacheExecutorService, VersionedTbCache cache) { this.attributesDao = attributesDao; this.jpaExecutorService = jpaExecutorService; + this.edqsService = edqsService; this.cacheExecutorService = cacheExecutorService; this.cache = cache; @@ -237,8 +243,10 @@ public class CachedAttributesService implements AttributesService { private ListenableFuture doSave(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); - return Futures.transform(future, version -> { - put(entityId, scope, new BaseAttributeKvEntry(((BaseAttributeKvEntry)attribute).getKv(), attribute.getLastUpdateTs(), version)); + return Futures.transform(future, version -> { + BaseAttributeKvEntry attributeKvEntry = new BaseAttributeKvEntry(((BaseAttributeKvEntry) attribute).getKv(), attribute.getLastUpdateTs(), version); + put(entityId, scope, attributeKvEntry); + edqsService.onUpdate(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, attributeKvEntry, version)); return version; }, cacheExecutor); } @@ -256,7 +264,11 @@ public class CachedAttributesService implements AttributesService { List>> futures = attributesDao.removeAllWithVersions(tenantId, entityId, scope, attributeKeys); return Futures.allAsList(futures.stream().map(future -> Futures.transform(future, keyVersionPair -> { String key = keyVersionPair.getFirst(); - cache.evict(new AttributeCacheKey(scope, entityId, key), keyVersionPair.getSecond()); + Long version = keyVersionPair.getSecond(); + cache.evict(new AttributeCacheKey(scope, entityId, key), version); + if (version != null) { + edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, version)); + } return key; }, cacheExecutor)).collect(Collectors.toList())); } @@ -269,6 +281,8 @@ public class CachedAttributesService implements AttributesService { String key = deleted.getValue(); if (scope != null && key != null) { cache.evict(new AttributeCacheKey(scope, entityId, key)); + // using version as Long.MAX_VALUE because we expect that the entity is deleted and there won't be any attributes after this + edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, Long.MAX_VALUE)); } }); return result.size(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java new file mode 100644 index 0000000000..dceef73aba --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -0,0 +1,208 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.cf; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; +import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; +import org.thingsboard.server.dao.exception.IncorrectParameterException; +import org.thingsboard.server.dao.service.DataValidator; + +import java.util.List; +import java.util.Optional; + +import static org.thingsboard.server.dao.service.Validator.validateId; +import static org.thingsboard.server.dao.service.Validator.validatePageLink; + +@Service("CalculatedFieldDaoService") +@Slf4j +@RequiredArgsConstructor +public class BaseCalculatedFieldService extends AbstractEntityService implements CalculatedFieldService { + + public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; + public static final String INCORRECT_CALCULATED_FIELD_ID = "Incorrect calculatedFieldId "; + public static final String INCORRECT_ENTITY_ID = "Incorrect entityId "; + + private final CalculatedFieldDao calculatedFieldDao; + private final CalculatedFieldLinkDao calculatedFieldLinkDao; + private final DataValidator calculatedFieldDataValidator; + private final DataValidator calculatedFieldLinkDataValidator; + + @Override + public CalculatedField save(CalculatedField calculatedField) { + CalculatedField oldCalculatedField = calculatedFieldDataValidator.validate(calculatedField, CalculatedField::getTenantId); + try { + TenantId tenantId = calculatedField.getTenantId(); + log.trace("Executing save calculated field, [{}]", calculatedField); + updateDebugSettings(tenantId, calculatedField, System.currentTimeMillis()); + CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); + createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); + eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedCalculatedField.getTenantId()).entityId(savedCalculatedField.getId()) + .entity(savedCalculatedField).oldEntity(oldCalculatedField).created(calculatedField.getId() == null).build()); + return savedCalculatedField; + } catch (Exception e) { + checkConstraintViolation(e, + "calculated_field_unq_key", "Calculated Field with such name is already in exists!", + "calculated_field_external_id_unq_key", "Calculated Field with such external id already exists!"); + throw e; + } + } + + @Override + public CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing findById, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedFieldId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); + return calculatedFieldDao.findById(tenantId, calculatedFieldId.getId()); + } + + @Override + public List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing findCalculatedFieldIdsByEntityId [{}]", entityId); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); + return calculatedFieldDao.findCalculatedFieldIdsByEntityId(tenantId, entityId); + } + + @Override + public List findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing findCalculatedFieldsByEntityId [{}]", entityId); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); + return calculatedFieldDao.findCalculatedFieldsByEntityId(tenantId, entityId); + } + + @Override + public PageData findAllCalculatedFields(PageLink pageLink) { + log.trace("Executing findAll, pageLink [{}]", pageLink); + validatePageLink(pageLink); + return calculatedFieldDao.findAll(pageLink); + } + + @Override + public PageData findAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink) { + log.trace("Executing findAllByEntityId, entityId [{}], pageLink [{}]", entityId, pageLink); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); + validatePageLink(pageLink); + return calculatedFieldDao.findAllByEntityId(tenantId, entityId, pageLink); + } + + @Override + public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); + deleteEntity(tenantId, calculatedFieldId, false); + } + + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + CalculatedField calculatedField = calculatedFieldDao.findById(tenantId, id.getId()); + if (calculatedField == null) { + if (force) { + return; + } else { + throw new IncorrectParameterException("Unable to delete non-existent calculated field."); + } + } + deleteCalculatedField(tenantId, calculatedField); + } + + private void deleteCalculatedField(TenantId tenantId, CalculatedField calculatedField) { + log.trace("Executing deleteCalculatedField, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedField.getId()); + calculatedFieldDao.removeById(tenantId, calculatedField.getUuidId()); + eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(calculatedField.getId()).entity(calculatedField).build()); + } + + @Override + public int deleteAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing deleteAllCalculatedFieldsByEntityId, tenantId [{}], entityId [{}]", tenantId, entityId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); + List calculatedFields = calculatedFieldDao.removeAllByEntityId(tenantId, entityId); + return calculatedFields.size(); + } + + @Override + public CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink) { + calculatedFieldLinkDataValidator.validate(calculatedFieldLink, CalculatedFieldLink::getTenantId); + log.trace("Executing save calculated field link, [{}]", calculatedFieldLink); + return calculatedFieldLinkDao.save(tenantId, calculatedFieldLink); + } + + @Override + public CalculatedFieldLink findCalculatedFieldLinkById(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId) { + log.trace("Executing findCalculatedFieldLinkById, tenantId [{}], calculatedFieldLinkId [{}]", tenantId, calculatedFieldLinkId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(calculatedFieldLinkId, id -> "Incorrect calculatedFieldLinkId " + id); + return calculatedFieldLinkDao.findById(tenantId, calculatedFieldLinkId.getId()); + } + + @Override + public List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing findAllCalculatedFieldLinksById, calculatedFieldId [{}]", calculatedFieldId); + return calculatedFieldLinkDao.findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId); + } + + @Override + public List findAllCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing findAllCalculatedFieldLinksByEntityId, entityId [{}]", entityId); + return calculatedFieldLinkDao.findCalculatedFieldLinksByEntityId(tenantId, entityId); + } + + @Override + public PageData findAllCalculatedFieldLinks(PageLink pageLink) { + log.trace("Executing findAllCalculatedFieldLinks, pageLink [{}]", pageLink); + validatePageLink(pageLink); + return calculatedFieldLinkDao.findAll(pageLink); + } + + @Override + public boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId) { + return calculatedFieldDao.findAllByTenantId(tenantId).stream() + .filter(calculatedField -> !referencedEntityId.equals(calculatedField.getEntityId())) + .map(CalculatedField::getConfiguration) + .map(CalculatedFieldConfiguration::getReferencedEntities) + .anyMatch(referencedEntities -> referencedEntities.contains(referencedEntityId)); + } + + @Override + public Optional> findEntity(TenantId tenantId, EntityId entityId) { + return Optional.ofNullable(findById(tenantId, new CalculatedFieldId(entityId.getId()))); + } + + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD; + } + + private void createOrUpdateCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { + List links = calculatedField.getConfiguration().buildCalculatedFieldLinks(tenantId, calculatedField.getEntityId(), calculatedField.getId()); + links.forEach(link -> saveCalculatedFieldLink(tenantId, link)); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java new file mode 100644 index 0000000000..a966977968 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.cf; + +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.Dao; + +import java.util.List; + +public interface CalculatedFieldDao extends Dao { + + List findAllByTenantId(TenantId tenantId); + + List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId); + + List findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + + List findAll(); + + PageData findAll(PageLink pageLink); + + PageData findAllByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink); + + List removeAllByEntityId(TenantId tenantId, EntityId entityId); + + long countCFByEntityId(TenantId tenantId, EntityId entityId); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java new file mode 100644 index 0000000000..8b4a5e7086 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.cf; + +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.Dao; + +import java.util.List; + +public interface CalculatedFieldLinkDao extends Dao { + + List findCalculatedFieldLinksByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + List findCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + + List findAll(); + + PageData findAll(PageLink pageLink); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/config/DedicatedEventsJpaDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/config/DedicatedEventsJpaDaoConfig.java index 5c24c45c16..258bac8dbd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/config/DedicatedEventsJpaDaoConfig.java +++ b/dao/src/main/java/org/thingsboard/server/dao/config/DedicatedEventsJpaDaoConfig.java @@ -29,6 +29,7 @@ import org.springframework.orm.jpa.JpaTransactionManager; import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.server.dao.model.sql.AuditLogEntity; +import org.thingsboard.server.dao.model.sql.CalculatedFieldDebugEventEntity; import org.thingsboard.server.dao.model.sql.ErrorEventEntity; import org.thingsboard.server.dao.model.sql.LifecycleEventEntity; import org.thingsboard.server.dao.model.sql.RuleChainDebugEventEntity; @@ -68,7 +69,7 @@ public class DedicatedEventsJpaDaoConfig { EntityManagerFactoryBuilder builder) { return builder .dataSource(eventsDataSource) - .packages(LifecycleEventEntity.class, StatisticsEventEntity.class, ErrorEventEntity.class, RuleNodeDebugEventEntity.class, RuleChainDebugEventEntity.class, AuditLogEntity.class) + .packages(LifecycleEventEntity.class, StatisticsEventEntity.class, ErrorEventEntity.class, RuleNodeDebugEventEntity.class, RuleChainDebugEventEntity.class, AuditLogEntity.class, CalculatedFieldDebugEventEntity.class) .persistenceUnit(EVENTS_PERSISTENCE_UNIT) .build(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java index b0a23bf6fc..c6bfa970a0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java @@ -30,7 +30,7 @@ import java.util.UUID; /** * The Interface CustomerDao. */ -public interface CustomerDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface CustomerDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Save or update customer object diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java index 76d05c2c98..4b05cdbc75 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java @@ -162,7 +162,7 @@ public class CustomerServiceImpl extends AbstractCachedEntityService, TenantEntityDao, ExportableEntityDao { +public interface DashboardDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Save or update dashboard object diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardInfoDao.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardInfoDao.java index 088b9e8a1f..6a3ec3295e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardInfoDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardInfoDao.java @@ -20,13 +20,14 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ImageContainerDao; +import org.thingsboard.server.dao.ResourceContainerDao; import java.util.UUID; /** * The Interface DashboardInfoDao. */ -public interface DashboardInfoDao extends Dao, ImageContainerDao { +public interface DashboardInfoDao extends Dao, ImageContainerDao, ResourceContainerDao { /** * Find dashboards by tenantId and page link. diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java index b8fff17a50..1b21310237 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java @@ -172,7 +172,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb var saved = dashboardDao.save(tenantId, dashboard); publishEvictEvent(new DashboardTitleEvictEvent(saved.getId())); eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(tenantId) - .entityId(saved.getId()).created(dashboard.getId() == null).build()); + .entityId(saved.getId()).entity(saved).created(dashboard.getId() == null).build()); if (dashboard.getId() == null) { countService.publishCountEntityEvictEvent(tenantId, EntityType.DASHBOARD); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 099bf381fa..efc57119eb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -22,7 +22,10 @@ import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.DeviceInfoFilter; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; @@ -39,7 +42,7 @@ import java.util.UUID; * The Interface DeviceDao. * */ -public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find device info by id. @@ -85,6 +88,16 @@ public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntit */ PageData findDevicesByTenantIdAndType(UUID tenantId, String type, PageLink pageLink); + /** + * Find device ids by tenantId, type and page link. + * + * @param tenantId the tenantId + * @param deviceProfileId the deviceProfileId + * @param pageLink the page link + * @return the list of device objects + */ + PageData findDeviceIdsByTenantIdAndDeviceProfileId(UUID tenantId, UUID deviceProfileId, PageLink pageLink); + PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(UUID tenantId, UUID deviceProfileId, OtaPackageType type, @@ -218,5 +231,8 @@ public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntit PageData findDeviceIdInfos(PageLink pageLink); + PageData findProfileEntityIdInfos(PageLink pageLink); + PageData findDeviceInfosByFilter(DeviceInfoFilter filter, PageLink pageLink); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 61523fd626..2ebcf046a6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -180,13 +180,12 @@ public class DeviceProfileServiceImpl extends CachedVersionedEntityService findProfileEntityIdInfos(PageLink pageLink) { + log.trace("Executing findProfileEntityIdInfos, pageLink [{}]", pageLink); + validatePageLink(pageLink); + return deviceDao.findProfileEntityIdInfos(pageLink); + } + @Override public PageData findDevicesByTenantIdAndType(TenantId tenantId, String type, PageLink pageLink) { log.trace("Executing findDevicesByTenantIdAndType, tenantId [{}], type [{}], pageLink [{}]", tenantId, type, pageLink); @@ -395,6 +403,15 @@ public class DeviceServiceImpl extends CachedVersionedEntityService findDeviceIdsByTenantIdAndDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId, PageLink pageLink) { + log.trace("Executing findDeviceIdsByTenantIdAndType, tenantId [{}], deviceProfileId [{}], pageLink [{}]", tenantId, deviceProfileId, pageLink); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(deviceProfileId, id -> INCORRECT_DEVICE_PROFILE_ID + id); + validatePageLink(pageLink); + return deviceDao.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId.getId(), deviceProfileId.getId(), pageLink); + } + @Override public PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, diff --git a/dao/src/main/java/org/thingsboard/server/dao/dictionary/KeyDictionaryDao.java b/dao/src/main/java/org/thingsboard/server/dao/dictionary/KeyDictionaryDao.java index 162cde1f2c..7b1c03bd6e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dictionary/KeyDictionaryDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dictionary/KeyDictionaryDao.java @@ -16,10 +16,15 @@ package org.thingsboard.server.dao.dictionary; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; + public interface KeyDictionaryDao { Integer getOrSaveKeyId(String strKey); String getKey(Integer keyId); + PageData findAll(PageLink pageLink); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeDao.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeDao.java index c6d456c2d9..fdb9144ab2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeDao.java @@ -35,7 +35,7 @@ import java.util.UUID; * The Interface EdgeDao. * */ -public interface EdgeDao extends Dao, TenantEntityDao { +public interface EdgeDao extends Dao, TenantEntityDao { Edge save(TenantId tenantId, Edge edge); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java index 75fc713be3..7560c7fb76 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.exception.DataValidationException; @@ -66,6 +67,10 @@ public abstract class AbstractEntityService { @Autowired protected EntityViewService entityViewService; + @Lazy + @Autowired + protected CalculatedFieldService calculatedFieldService; + @Lazy @Autowired(required = false) protected EdgeService edgeService; diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index f228db2825..a762aabf38 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.HasLabel; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTitle; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.query.EdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsResponse; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; @@ -40,8 +42,10 @@ import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityFilterType; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.msg.edqs.EdqsApiService; import org.thingsboard.server.dao.exception.IncorrectParameterException; import java.util.ArrayList; @@ -50,10 +54,12 @@ import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ExecutionException; import java.util.function.Function; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.id.EntityId.NULL_UUID; +import static org.thingsboard.server.common.data.query.EntityFilterType.ENTITY_TYPE; import static org.thingsboard.server.dao.service.Validator.validateEntityDataPageLink; import static org.thingsboard.server.dao.service.Validator.validateId; @@ -79,12 +85,24 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe @Lazy EntityServiceRegistry entityServiceRegistry; + @Autowired + @Lazy + private EdqsApiService edqsApiService; + @Override public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query) { log.trace("Executing countEntitiesByQuery, tenantId [{}], customerId [{}], query [{}]", tenantId, customerId, query); validateId(tenantId, id -> INCORRECT_TENANT_ID + id); validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityCountQuery(query); + + if (edqsApiService.isEnabled() && validForEdqs(query) && !tenantId.isSysTenantId()) { + EdqsRequest request = EdqsRequest.builder() + .entityCountQuery(query) + .build(); + EdqsResponse response = processEdqsRequest(tenantId, customerId, request); + return response.getEntityCountQueryResult(); + } return this.entityQueryDao.countEntitiesByQuery(tenantId, customerId, query); } @@ -95,6 +113,14 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityDataQuery(query); + if (edqsApiService.isEnabled() && validForEdqs(query)) { + EdqsRequest request = EdqsRequest.builder() + .entityDataQuery(query) + .build(); + EdqsResponse response = processEdqsRequest(tenantId, customerId, request); + return response.getEntityDataQueryResult(); + } + if (!isValidForOptimization(query)) { return this.entityQueryDao.findEntityDataByQuery(tenantId, customerId, query); } @@ -110,6 +136,25 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe return new PageData<>(result, entityDataByQuery.getTotalPages(), entityDataByQuery.getTotalElements(), entityDataByQuery.hasNext()); } + private boolean validForEdqs(EntityCountQuery query) { // for compatibility with PE + return true; + } + + private EdqsResponse processEdqsRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) { + EdqsResponse response; + try { + log.debug("[{}] Sending request to EDQS: {}", tenantId, request); + response = edqsApiService.processRequest(tenantId, customerId, request).get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + log.debug("[{}] Received response from EDQS: {}", tenantId, response); + if (response.getError() != null) { + throw new RuntimeException(response.getError()); + } + return response; + } + @Override public Optional fetchEntityName(TenantId tenantId, EntityId entityId) { log.trace("Executing fetchEntityName [{}]", entityId); @@ -134,6 +179,11 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe return fetchAndConvert(tenantId, entityId, this::getNameLabelAndCustomerDetails); } + @Override + public Optional> fetchEntity(TenantId tenantId, EntityId entityId) { + return fetchAndConvert(tenantId, entityId, Function.identity()); + } + private Optional fetchAndConvert(TenantId tenantId, EntityId entityId, Function, T> converter) { EntityDaoService entityDaoService = entityServiceRegistry.getServiceByEntityType(entityId.getEntityType()); Optional> entityOpt = entityDaoService.findEntity(tenantId, entityId); @@ -184,6 +234,8 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe throw new IncorrectParameterException("Query entity filter type must be specified."); } else if (query.getEntityFilter().getType().equals(EntityFilterType.RELATIONS_QUERY)) { validateRelationQuery((RelationsQueryFilter) query.getEntityFilter()); + } else if (query.getEntityFilter().getType().equals(ENTITY_TYPE)) { + validateEntityTypeQuery((EntityTypeFilter) query.getEntityFilter()); } } @@ -192,6 +244,12 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateEntityDataPageLink(query.getPageLink()); } + private static void validateEntityTypeQuery(EntityTypeFilter filter) { + if (filter.getEntityType() == null) { + throw new IncorrectParameterException("Entity type is required"); + } + } + private static void validateRelationQuery(RelationsQueryFilter queryFilter) { if (queryFilter.isMultiRoot() && queryFilter.getMultiRootEntitiesType() == null) { throw new IncorrectParameterException("Multi-root relation query filter should contain 'multiRootEntitiesType'"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java b/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java index a795bddffa..9c50ad621f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java @@ -43,6 +43,9 @@ public class DefaultEntityServiceRegistry implements EntityServiceRegistry { if (EntityType.RULE_CHAIN.equals(entityType)) { entityDaoServicesMap.put(EntityType.RULE_NODE, entityDaoService); } + if (EntityType.CALCULATED_FIELD.equals(entityType)) { + entityDaoServicesMap.put(EntityType.CALCULATED_FIELD_LINK, entityDaoService); + } }); log.debug("Initialized EntityServiceRegistry total [{}] entries", entityDaoServicesMap.size()); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java index 74deb29247..2dbda352b4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java @@ -26,6 +26,7 @@ import java.util.Map; @Service @Slf4j +@SuppressWarnings({"unchecked"}) public class EntityDaoRegistry { private final Map> daos = new EnumMap<>(EntityType.class); @@ -39,7 +40,6 @@ public class EntityDaoRegistry { }); } - @SuppressWarnings("unchecked") public Dao getDao(EntityType entityType) { Dao dao = (Dao) daos.get(entityType); if (dao == null) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index 867264cfaf..0e742e20db 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -123,7 +123,7 @@ public class EntityViewServiceImpl extends CachedVersionedEntityService { private final T oldEntity; private final EntityId entityId; private final Boolean created; + private final Boolean broadcastEvent; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java index 120287fb83..1ca2973936 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java @@ -75,6 +75,7 @@ public class CleanUpService { submitTask(HousekeeperTask.deleteTelemetry(tenantId, entityId)); submitTask(HousekeeperTask.deleteEvents(tenantId, entityId)); submitTask(HousekeeperTask.deleteAlarms(tenantId, entityId)); + submitTask(HousekeeperTask.deleteCalculatedFields(tenantId, entityId)); } public void removeTenantEntities(TenantId tenantId, EntityType... entityTypes) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index aaf9659f1b..148908d063 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -376,6 +376,7 @@ public class ModelConstants { public static final String STATS_EVENT_TABLE_NAME = "stats_event"; public static final String RULE_NODE_DEBUG_EVENT_TABLE_NAME = "rule_node_debug_event"; public static final String RULE_CHAIN_DEBUG_EVENT_TABLE_NAME = "rule_chain_debug_event"; + public static final String CALCULATED_FIELD_DEBUG_EVENT_TABLE_NAME = "cf_debug_event"; public static final String EVENT_TENANT_ID_PROPERTY = TENANT_ID_PROPERTY; public static final String EVENT_SERVICE_ID_PROPERTY = "service_id"; @@ -400,6 +401,10 @@ public class ModelConstants { public static final String EVENT_METADATA_COLUMN_NAME = "e_metadata"; public static final String EVENT_MESSAGE_COLUMN_NAME = "e_message"; + public static final String EVENT_CALCULATED_FIELD_ID_COLUMN_NAME = "cf_id"; + public static final String EVENT_CALCULATED_FIELD_ARGUMENTS_COLUMN_NAME = "e_args"; + public static final String EVENT_CALCULATED_FIELD_RESULT_COLUMN_NAME = "e_result"; + public static final String DEBUG_MODE = "debug_mode"; public static final String DEBUG_SETTINGS = "debug_settings"; public static final String SINGLETON_MODE = "singleton_mode"; @@ -712,6 +717,28 @@ public class ModelConstants { public static final String QR_CODE_SETTINGS_BUNDLE_ID_PROPERTY = "mobile_app_bundle_id"; public static final String QR_CODE_SETTINGS_CONFIG_PROPERTY = "qr_code_config"; + /** + * Calculated fields constants. + */ + public static final String CALCULATED_FIELD_TABLE_NAME = "calculated_field"; + public static final String CALCULATED_FIELD_TENANT_ID_COLUMN = TENANT_ID_COLUMN; + public static final String CALCULATED_FIELD_ENTITY_TYPE = ENTITY_TYPE_COLUMN; + public static final String CALCULATED_FIELD_ENTITY_ID = ENTITY_ID_COLUMN; + public static final String CALCULATED_FIELD_TYPE = "type"; + public static final String CALCULATED_FIELD_NAME = "name"; + public static final String CALCULATED_FIELD_CONFIGURATION_VERSION = "configuration_version"; + public static final String CALCULATED_FIELD_CONFIGURATION = "configuration"; + public static final String CALCULATED_FIELD_VERSION = "version"; + + /** + * Calculated field links constants. + */ + public static final String CALCULATED_FIELD_LINK_TABLE_NAME = "calculated_field_link"; + public static final String CALCULATED_FIELD_LINK_TENANT_ID_COLUMN = TENANT_ID_COLUMN; + public static final String CALCULATED_FIELD_LINK_ENTITY_TYPE = ENTITY_TYPE_COLUMN; + public static final String CALCULATED_FIELD_LINK_ENTITY_ID = ENTITY_ID_COLUMN; + public static final String CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID = "calculated_field_id"; + protected static final String[] NONE_AGGREGATION_COLUMNS = new String[]{LONG_VALUE_COLUMN, DOUBLE_VALUE_COLUMN, BOOLEAN_VALUE_COLUMN, STRING_VALUE_COLUMN, JSON_VALUE_COLUMN, KEY_COLUMN, TS_COLUMN}; protected static final String[] COUNT_AGGREGATION_COLUMNS = new String[]{count(LONG_VALUE_COLUMN), count(DOUBLE_VALUE_COLUMN), count(BOOLEAN_VALUE_COLUMN), count(STRING_VALUE_COLUMN), count(JSON_VALUE_COLUMN), max(TS_COLUMN)}; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldDebugEventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldDebugEventEntity.java new file mode 100644 index 0000000000..cf771238b9 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldDebugEventEntity.java @@ -0,0 +1,104 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Table; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; + +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_DEBUG_EVENT_TABLE_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_CALCULATED_FIELD_ARGUMENTS_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_CALCULATED_FIELD_ID_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_CALCULATED_FIELD_RESULT_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_ID_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_TYPE_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERROR_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MSG_ID_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MSG_TYPE_COLUMN_NAME; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = CALCULATED_FIELD_DEBUG_EVENT_TABLE_NAME) +@NoArgsConstructor +public class CalculatedFieldDebugEventEntity extends EventEntity implements BaseEntity { + + @Column(name = EVENT_CALCULATED_FIELD_ID_COLUMN_NAME) + private UUID calculatedFieldId; + @Column(name = EVENT_ENTITY_ID_COLUMN_NAME) + private UUID eventEntityId; + @Column(name = EVENT_ENTITY_TYPE_COLUMN_NAME) + private String eventEntityType; + @Column(name = EVENT_MSG_ID_COLUMN_NAME) + private UUID msgId; + @Column(name = EVENT_MSG_TYPE_COLUMN_NAME) + private String msgType; + @Column(name = EVENT_CALCULATED_FIELD_ARGUMENTS_COLUMN_NAME) + private String arguments; + @Column(name = EVENT_CALCULATED_FIELD_RESULT_COLUMN_NAME) + private String result; + @Column(name = EVENT_ERROR_COLUMN_NAME) + private String error; + + public CalculatedFieldDebugEventEntity(CalculatedFieldDebugEvent event) { + super(event); + if (event.getCalculatedFieldId() != null) { + this.calculatedFieldId = event.getCalculatedFieldId().getId(); + } + if (event.getEventEntity() != null) { + this.eventEntityId = event.getEventEntity().getId(); + this.eventEntityType = event.getEventEntity().getEntityType().name(); + } + this.msgId = event.getMsgId(); + this.msgType = event.getMsgType(); + this.arguments = event.getArguments(); + this.result = event.getResult(); + this.error = event.getError(); + } + + @Override + public CalculatedFieldDebugEvent toData() { + var builder = CalculatedFieldDebugEvent.builder() + .id(id) + .tenantId(TenantId.fromUUID(tenantId)) + .ts(ts) + .serviceId(serviceId) + .entityId(entityId) + .msgId(msgId) + .msgType(msgType) + .arguments(arguments) + .result(result) + .error(error); + if (calculatedFieldId != null) { + builder.calculatedFieldId(new CalculatedFieldId(calculatedFieldId)); + } + if (eventEntityId != null) { + builder.eventEntity(EntityIdFactory.getByTypeAndUuid(eventEntityType, eventEntityId)); + } + return builder.build(); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java new file mode 100644 index 0000000000..de6a1365b1 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -0,0 +1,117 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import com.fasterxml.jackson.databind.JsonNode; +import jakarta.persistence.Column; +import jakarta.persistence.Convert; +import jakarta.persistence.Entity; +import jakarta.persistence.Table; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; +import org.thingsboard.server.dao.model.BaseVersionedEntity; +import org.thingsboard.server.dao.util.mapping.JsonConverter; + +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION_VERSION; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_ID; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_TYPE; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TABLE_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TENANT_ID_COLUMN; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TYPE; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_VERSION; +import static org.thingsboard.server.dao.model.ModelConstants.DEBUG_SETTINGS; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = CALCULATED_FIELD_TABLE_NAME) +public class CalculatedFieldEntity extends BaseVersionedEntity implements BaseEntity { + + @Column(name = CALCULATED_FIELD_TENANT_ID_COLUMN) + private UUID tenantId; + + @Column(name = CALCULATED_FIELD_ENTITY_TYPE) + private String entityType; + + @Column(name = CALCULATED_FIELD_ENTITY_ID) + private UUID entityId; + + @Column(name = CALCULATED_FIELD_TYPE) + private String type; + + @Column(name = CALCULATED_FIELD_NAME) + private String name; + + @Column(name = CALCULATED_FIELD_CONFIGURATION_VERSION) + private int configurationVersion; + + @Convert(converter = JsonConverter.class) + @Column(name = CALCULATED_FIELD_CONFIGURATION) + private JsonNode configuration; + + @Column(name = CALCULATED_FIELD_VERSION) + private Long version; + + @Column(name = DEBUG_SETTINGS) + private String debugSettings; + + public CalculatedFieldEntity() { + super(); + } + + public CalculatedFieldEntity(CalculatedField calculatedField) { + this.setUuid(calculatedField.getUuidId()); + this.createdTime = calculatedField.getCreatedTime(); + this.tenantId = calculatedField.getTenantId().getId(); + this.entityType = calculatedField.getEntityId().getEntityType().name(); + this.entityId = calculatedField.getEntityId().getId(); + this.type = calculatedField.getType().name(); + this.name = calculatedField.getName(); + this.configurationVersion = calculatedField.getConfigurationVersion(); + this.configuration = JacksonUtil.valueToTree(calculatedField.getConfiguration()); + this.version = calculatedField.getVersion(); + this.debugSettings = JacksonUtil.toString(calculatedField.getDebugSettings()); + } + + @Override + public CalculatedField toData() { + CalculatedField calculatedField = new CalculatedField(new CalculatedFieldId(id)); + calculatedField.setCreatedTime(createdTime); + calculatedField.setTenantId(TenantId.fromUUID(tenantId)); + calculatedField.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedField.setType(CalculatedFieldType.valueOf(type)); + calculatedField.setName(name); + calculatedField.setConfigurationVersion(configurationVersion); + calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); + calculatedField.setVersion(version); + calculatedField.setDebugSettings(JacksonUtil.fromString(debugSettings, DebugSettings.class)); + return calculatedField; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java new file mode 100644 index 0000000000..0f2a6455ec --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -0,0 +1,79 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Table; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; +import org.thingsboard.server.dao.model.BaseSqlEntity; + +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_ID; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_TYPE; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_TABLE_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_TENANT_ID_COLUMN; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = CALCULATED_FIELD_LINK_TABLE_NAME) +public class CalculatedFieldLinkEntity extends BaseSqlEntity implements BaseEntity { + + @Column(name = CALCULATED_FIELD_LINK_TENANT_ID_COLUMN) + private UUID tenantId; + + @Column(name = CALCULATED_FIELD_LINK_ENTITY_TYPE) + private String entityType; + + @Column(name = CALCULATED_FIELD_LINK_ENTITY_ID) + private UUID entityId; + + @Column(name = CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID) + private UUID calculatedFieldId; + + public CalculatedFieldLinkEntity() { + super(); + } + + public CalculatedFieldLinkEntity(CalculatedFieldLink calculatedFieldLink) { + super(calculatedFieldLink); + this.tenantId = calculatedFieldLink.getTenantId().getId(); + this.entityType = calculatedFieldLink.getEntityId().getEntityType().name(); + this.entityId = calculatedFieldLink.getEntityId().getId(); + this.calculatedFieldId = calculatedFieldLink.getCalculatedFieldId().getId(); + } + + @Override + public CalculatedFieldLink toData() { + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(new CalculatedFieldLinkId(id)); + calculatedFieldLink.setCreatedTime(createdTime); + calculatedFieldLink.setTenantId(TenantId.fromUUID(tenantId)); + calculatedFieldLink.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedFieldLink.setCalculatedFieldId(new CalculatedFieldId(calculatedFieldId)); + return calculatedFieldLink; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java index 782872b12f..b3870a9e3a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java @@ -26,6 +26,7 @@ import jakarta.persistence.SqlResultSetMapping; import jakarta.persistence.SqlResultSetMappings; import jakarta.persistence.Table; import lombok.Data; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; import org.thingsboard.server.dao.sqlts.latest.SearchTsKvLatestRepository; @@ -91,4 +92,12 @@ public final class TsKvLatestEntity extends AbstractTsKvEntity { this.strKey = strKey; this.version = version; } + + @Override + public TsKvEntry toData() { + TsKvEntry tsKvEntry = super.toData(); + tsKvEntry.setVersion(version); + return tsKvEntry; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTargetDao.java b/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTargetDao.java index 7505d314d4..eeae618c61 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTargetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTargetDao.java @@ -28,7 +28,7 @@ import org.thingsboard.server.dao.TenantEntityDao; import java.util.List; -public interface NotificationTargetDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface NotificationTargetDao extends Dao, TenantEntityDao, ExportableEntityDao { PageData findByTenantIdAndPageLink(TenantId tenantId, PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java index bc072567d6..f8f877e55e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java @@ -21,5 +21,7 @@ import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.TenantEntityWithDataDao; public interface OtaPackageDao extends Dao, TenantEntityWithDataDao { + Long sumDataSizeByTenantId(TenantId tenantId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java index 87c2b513a2..9e4c7136d5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.queue.QueueStats; import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; +import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.Validator; @@ -51,7 +53,10 @@ public class BaseQueueStatsService extends AbstractEntityService implements Queu public QueueStats save(TenantId tenantId, QueueStats queueStats) { log.trace("Executing save [{}]", queueStats); queueStatsValidator.validate(queueStats, QueueStats::getTenantId); - return queueStatsDao.save(tenantId, queueStats); + QueueStats savedQueueStats = queueStatsDao.save(tenantId, queueStats); + eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedQueueStats.getTenantId()).entityId(savedQueueStats.getId()) + .entity(savedQueueStats).created(queueStats.getId() == null).build()); + return savedQueueStats; } @Override @@ -80,7 +85,7 @@ public class BaseQueueStatsService extends AbstractEntityService implements Queu public PageData findByTenantId(TenantId tenantId, PageLink pageLink) { log.trace("Executing findByTenantId, tenantId: [{}]", tenantId); Validator.validatePageLink(pageLink); - return queueStatsDao.findByTenantId(tenantId, pageLink); + return queueStatsDao.findAllByTenantId(tenantId, pageLink); } @Override @@ -93,6 +98,7 @@ public class BaseQueueStatsService extends AbstractEntityService implements Queu @Override public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { queueStatsDao.removeById(tenantId, id.getId()); + eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(id).build()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java b/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java index c9b6df0016..cd1f0701fe 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java @@ -17,19 +17,16 @@ package org.thingsboard.server.dao.queue; import org.thingsboard.server.common.data.id.QueueStatsId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.queue.QueueStats; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.TenantEntityDao; import java.util.List; -public interface QueueStatsDao extends Dao { +public interface QueueStatsDao extends Dao, TenantEntityDao { QueueStats findByTenantIdQueueNameAndServiceId(TenantId tenantId, String queueName, String serviceId); - PageData findByTenantId(TenantId tenantId, PageLink pageLink); - void deleteByTenantId(TenantId tenantId); List findByIds(TenantId tenantId, List queueStatsIds); diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java index f8fe55cf02..e91c1939f6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java @@ -75,8 +75,6 @@ import static org.thingsboard.server.common.data.StringUtils.isNotEmpty; @Slf4j public class BaseImageService extends BaseResourceService implements ImageService { - private static final int MAX_ENTITIES_TO_FIND = 10; - public static Map DASHBOARD_BASE64_MAPPING = new HashMap<>(); public static Map WIDGET_TYPE_BASE64_MAPPING = new HashMap<>(); @@ -107,19 +105,15 @@ public class BaseImageService extends BaseResourceService implements ImageServic private final AssetProfileDao assetProfileDao; private final DeviceProfileDao deviceProfileDao; private final WidgetsBundleDao widgetsBundleDao; - private final WidgetTypeDao widgetTypeDao; - private final DashboardInfoDao dashboardInfoDao; private final Map> imageContainerDaoMap = new HashMap<>(); public BaseImageService(TbResourceDao resourceDao, TbResourceInfoDao resourceInfoDao, ResourceDataValidator resourceValidator, AssetProfileDao assetProfileDao, DeviceProfileDao deviceProfileDao, WidgetsBundleDao widgetsBundleDao, WidgetTypeDao widgetTypeDao, DashboardInfoDao dashboardInfoDao) { - super(resourceDao, resourceInfoDao, resourceValidator); + super(resourceDao, resourceInfoDao, resourceValidator, widgetTypeDao, dashboardInfoDao); this.assetProfileDao = assetProfileDao; this.deviceProfileDao = deviceProfileDao; this.widgetsBundleDao = widgetsBundleDao; - this.widgetTypeDao = widgetTypeDao; - this.dashboardInfoDao = dashboardInfoDao; } @PostConstruct @@ -131,7 +125,6 @@ public class BaseImageService extends BaseResourceService implements ImageServic imageContainerDaoMap.put(EntityType.DASHBOARD, dashboardInfoDao); } - @Override @SneakyThrows public TbResourceInfo saveImage(TbResource image) { @@ -311,7 +304,8 @@ public class BaseImageService extends BaseResourceService implements ImageServic } } if (success) { - deleteResource(tenantId, imageId, force); + success = deleteResource(tenantId, imageId, true) + .isSuccess(); } return result.success(success).build(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java index 76aefdbb93..bf73c59708 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.hash.Hashing; import com.google.common.util.concurrent.ListenableFuture; +import jakarta.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -39,6 +40,7 @@ import org.thingsboard.server.common.data.ResourceExportData; import org.thingsboard.server.common.data.ResourceSubType; import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TbResourceDeleteResult; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.TbResourceInfoFilter; import org.thingsboard.server.common.data.id.EntityId; @@ -48,6 +50,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; +import org.thingsboard.server.dao.ResourceContainerDao; +import org.thingsboard.server.dao.dashboard.DashboardInfoDao; import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; @@ -55,6 +59,7 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; import org.thingsboard.server.dao.service.validator.ResourceDataValidator; +import org.thingsboard.server.dao.widget.WidgetTypeDao; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -85,6 +90,17 @@ public class BaseResourceService extends AbstractCachedEntityService> resourceContainerDaoMap = new HashMap<>(); + protected static final int MAX_ENTITIES_TO_FIND = 10; + + @PostConstruct + public void init() { + resourceContainerDaoMap.put(EntityType.WIDGET_TYPE, widgetTypeDao); + resourceContainerDaoMap.put(EntityType.DASHBOARD, dashboardInfoDao); + } + @Autowired @Lazy private ImageService imageService; @@ -313,23 +329,45 @@ public class BaseResourceService extends AbstractCachedEntityService INCORRECT_RESOURCE_ID + id); TbResourceInfo resource = findResourceInfoById(tenantId, resourceId); + boolean success = true; + var result = TbResourceDeleteResult.builder(); + if (resource == null) { - return; + if (!force) { + success = false; + } + return result.success(success).build(); } + if (!force) { - resourceValidator.validateDelete(tenantId, resource); + if (resource.getResourceType() == ResourceType.JS_MODULE) { + var link = resource.getLink(); + Map>> affectedEntities = new HashMap<>(); + + resourceContainerDaoMap.forEach((entityType, resourceContainerDao) -> { + var entities = tenantId.isSysTenantId() ? resourceContainerDao.findByResourceLink(link, MAX_ENTITIES_TO_FIND) : + resourceContainerDao.findByTenantIdAndResourceLink(tenantId, link, MAX_ENTITIES_TO_FIND); + if (!entities.isEmpty()) { + affectedEntities.put(entityType.name(), entities); + } + }); + + if (!affectedEntities.isEmpty()) { + success = false; + result.references(affectedEntities); + } + } } - resourceDao.removeById(tenantId, resourceId.getId()); - eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entity(resource).entityId(resourceId).build()); + if (success) { + resourceDao.removeById(tenantId, resourceId.getId()); + eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entity(resource).entityId(resourceId).build()); + } + + return result.success(success).build(); } @Override @@ -666,7 +704,7 @@ public class BaseResourceService extends AbstractCachedEntityService, TenantEntityDao, ExportableEntityDao { +public interface RuleChainDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find rule chains by tenantId and page link. diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java new file mode 100644 index 0000000000..187de20667 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -0,0 +1,74 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.validator; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.dao.cf.CalculatedFieldDao; +import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.service.DataValidator; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; + +@Component +public class CalculatedFieldDataValidator extends DataValidator { + + @Autowired + private CalculatedFieldDao calculatedFieldDao; + + @Autowired + private ApiLimitService apiLimitService; + + @Override + protected void validateCreate(TenantId tenantId, CalculatedField calculatedField) { + validateNumberOfCFsPerEntity(tenantId, calculatedField.getEntityId()); + validateNumberOfArgumentsPerCF(tenantId, calculatedField); + } + + @Override + protected CalculatedField validateUpdate(TenantId tenantId, CalculatedField calculatedField) { + CalculatedField old = calculatedFieldDao.findById(calculatedField.getTenantId(), calculatedField.getId().getId()); + if (old == null) { + throw new DataValidationException("Can't update non existing calculated field!"); + } + validateNumberOfArgumentsPerCF(tenantId, calculatedField); + return old; + } + + private void validateNumberOfCFsPerEntity(TenantId tenantId, EntityId entityId) { + long maxCFsPerEntity = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxCalculatedFieldsPerEntity); + if (maxCFsPerEntity <= 0) { + return; + } + if (calculatedFieldDao.countCFByEntityId(tenantId, entityId) >= maxCFsPerEntity) { + throw new DataValidationException("Calculated fields per entity limit reached!"); + } + } + + private void validateNumberOfArgumentsPerCF(TenantId tenantId, CalculatedField calculatedField) { + long maxArgumentsPerCF = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxArgumentsPerCF); + if (maxArgumentsPerCF <= 0) { + return; + } + if (calculatedField.getConfiguration().getArguments().size() > maxArgumentsPerCF) { + throw new DataValidationException("Calculated field arguments limit reached!"); + } + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java new file mode 100644 index 0000000000..aaba200c92 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.validator; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.cf.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.service.DataValidator; + +@Component +public class CalculatedFieldLinkDataValidator extends DataValidator { + + @Autowired + private CalculatedFieldLinkDao calculatedFieldLinkDao; + + @Override + protected CalculatedFieldLink validateUpdate(TenantId tenantId, CalculatedFieldLink calculatedFieldLink) { + CalculatedFieldLink old = calculatedFieldLinkDao.findById(calculatedFieldLink.getTenantId(), calculatedFieldLink.getId().getId()); + if (old == null) { + throw new DataValidationException("Can't update non existing calculated field link!"); + } + return old; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java index 2f49e7008b..f9f03adeff 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java @@ -21,7 +21,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.TbResource; -import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.id.TbResourceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; @@ -30,9 +29,6 @@ import org.thingsboard.server.dao.resource.TbResourceDao; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.dao.widget.WidgetTypeDao; - -import java.util.List; import static org.thingsboard.server.common.data.EntityType.TB_RESOURCE; @@ -42,9 +38,6 @@ public class ResourceDataValidator extends DataValidator { @Autowired private TbResourceDao resourceDao; - @Autowired - private WidgetTypeDao widgetTypeDao; - @Autowired private TenantService tenantService; @@ -111,12 +104,4 @@ public class ResourceDataValidator extends DataValidator { validateMaxSumDataSizePerTenant(tenantId, resourceDao, maxSumResourcesDataInBytes, dataSize, TB_RESOURCE); } } - - public void validateDelete(TenantId tenantId, TbResourceInfo resourceInfo) { - List widgets = widgetTypeDao.findWidgetTypesNamesByTenantIdAndResourceLink(tenantId.getId(), resourceInfo.getLink()); - if (!widgets.isEmpty()) { - throw new DataValidationException("Following widget types use this resource: " + String.join(", ", widgets)); - } - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java index 0f2550b7a2..05577b68e1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java @@ -88,10 +88,10 @@ public abstract class JpaAbstractDao, D> boolean flushed = false; EntityManager entityManager = getEntityManager(); if (isNew) { + entityManager.persist(entity); if (entity instanceof HasVersion versionedEntity) { versionedEntity.setVersion(1L); } - entityManager.persist(entity); } else { if (entity instanceof HasVersion versionedEntity) { if (versionedEntity.getVersion() == null) { @@ -106,23 +106,25 @@ public abstract class JpaAbstractDao, D> } } versionedEntity = entityManager.merge(versionedEntity); + entity = (E) versionedEntity; /* * by default, Hibernate doesn't issue an update query and thus version increment * if the entity was not modified. to bypass this and always increment the version, we do it manually * */ versionedEntity.setVersion(versionedEntity.getVersion() + 1); - /* - * flushing and then removing the entity from the persistence context so that it is not affected - * by next flushes (e.g. when a transaction is committed) to avoid double version increment - * */ - entityManager.flush(); - entityManager.detach(versionedEntity); - flushed = true; - entity = (E) versionedEntity; } else { entity = entityManager.merge(entity); } } + if (entity instanceof HasVersion versionedEntity) { + /* + * flushing and then removing the entity from the persistence context so that it is not affected + * by next flushes (e.g. when a transaction is committed) to avoid double version increment + * */ + entityManager.flush(); + entityManager.detach(versionedEntity); + flushed = true; + } if (flush && !flushed) { entityManager.flush(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmCommentRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmCommentRepository.java index 93e2b0f2c3..1a24ff6abb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmCommentRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmCommentRepository.java @@ -35,5 +35,9 @@ public interface AlarmCommentRepository extends JpaRepository findAllByAlarmId(@Param("alarmId") UUID alarmId, - Pageable pageable); + Pageable pageable); + + @Query("SELECT c FROM AlarmCommentEntity c WHERE c.userId IN (SELECT u.id FROM UserEntity u WHERE u.tenantId = :tenantId)") + Page findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java index 9cfa7a63eb..b9c1abe416 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java @@ -414,4 +414,6 @@ public interface AlarmRepository extends JpaRepository { @Param("alarmSeverities") List alarmSeverities, int limit); + Page findByTenantId(UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java index fa97fe258e..51eeb1c21d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.alarm; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; @@ -42,4 +44,6 @@ public interface EntityAlarmRepository extends JpaRepository findAllByEntityId(UUID entityId); + Page findByTenantId(UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java index 0fb7220784..42c9d68b01 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.alarm.AlarmCommentDao; import org.thingsboard.server.dao.model.sql.AlarmCommentEntity; import org.thingsboard.server.dao.sql.JpaPartitionedAbstractDao; @@ -44,7 +45,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.ALARM_COMMENT_TABL @Component @SqlDao @RequiredArgsConstructor -public class JpaAlarmCommentDao extends JpaPartitionedAbstractDao implements AlarmCommentDao { +public class JpaAlarmCommentDao extends JpaPartitionedAbstractDao implements AlarmCommentDao, TenantEntityDao { private final SqlPartitioningRepository partitioningRepository; @Value("${sql.alarm_comments.partition_size:168}") private int partitionSizeInHours; @@ -76,6 +77,11 @@ public class JpaAlarmCommentDao extends JpaPartitionedAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(alarmCommentRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + @Override protected Class getEntityClass() { return AlarmCommentEntity.class; @@ -85,4 +91,5 @@ public class JpaAlarmCommentDao extends JpaPartitionedAbstractDao getRepository() { return alarmCommentRepository; } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java index 1672ab4b3a..c21d8ae928 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java @@ -57,6 +57,7 @@ import org.thingsboard.server.common.data.query.AlarmDataQuery; import org.thingsboard.server.common.data.query.OriginatorAlarmFilter; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.alarm.AlarmDao; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.sql.AlarmEntity; @@ -84,7 +85,7 @@ import static org.thingsboard.server.dao.DaoUtil.toPageable; @Slf4j @Component @SqlDao -public class JpaAlarmDao extends JpaAbstractDao implements AlarmDao { +public class JpaAlarmDao extends JpaAbstractDao implements AlarmDao, TenantEntityDao { @Autowired private AlarmRepository alarmRepository; @@ -415,8 +416,8 @@ public class JpaAlarmDao extends JpaAbstractDao implements A } @Override - public long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query) { - return alarmQueryRepository.countAlarmsByQuery(tenantId, customerId, query); + public long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query, Collection orderedEntityIds) { + return alarmQueryRepository.countAlarmsByQuery(tenantId, customerId, query, orderedEntityIds); } @Override @@ -551,6 +552,11 @@ public class JpaAlarmDao extends JpaAbstractDao implements A } } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(alarmRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.ALARM; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaEntityAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaEntityAlarmDao.java new file mode 100644 index 0000000000..796fe63da8 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaEntityAlarmDao.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.alarm; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.alarm.EntityAlarm; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; +import org.thingsboard.server.dao.util.SqlDao; + +@Component +@SqlDao +public class JpaEntityAlarmDao implements TenantEntityDao { + + @Autowired + private EntityAlarmRepository entityAlarmRepository; + + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(entityAlarmRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink, "entityId", "alarmId"))); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetProfileRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetProfileRepository.java index fbcfbbda8a..eb35a4e18e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetProfileRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetProfileRepository.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.sql.asset; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; @@ -22,6 +23,7 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.asset.AssetProfileInfo; +import org.thingsboard.server.common.data.edqs.fields.AssetProfileFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.AssetProfileEntity; @@ -81,4 +83,8 @@ public interface AssetProfileRepository extends JpaRepository findAllTenantAssetProfileNames(@Param("tenantId") UUID tenantId); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.AssetProfileFields(a.id, a.createdTime, a.tenantId," + + "a.name, a.version, a.isDefault) FROM AssetProfileEntity a WHERE a.id > :id ORDER BY a.id") + List findNextBatch(@Param("id") UUID id, Limit limit); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java index 2cbdb0b90b..e475864684 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.dao.sql.asset; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.AssetFields; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.AssetEntity; @@ -139,6 +141,15 @@ public interface AssetRepository extends JpaRepository, Expor @Param("textSearch") String textSearch, Pageable pageable); + @Query("SELECT a.id FROM AssetEntity a " + + "WHERE a.tenantId = :tenantId " + + "AND a.assetProfileId = :assetProfileId " + + "AND (:textSearch IS NULL OR ilike(a.type, CONCAT('%', :textSearch, '%')) = true) ") + Page findAssetIdsByTenantIdAndAssetProfileId(@Param("tenantId") UUID tenantId, + @Param("assetProfileId") UUID assetProfileId, + @Param("textSearch") String textSearch, + Pageable pageable); + @Query("SELECT a FROM AssetEntity a WHERE a.tenantId = :tenantId " + "AND a.customerId = :customerId AND a.type = :type " + @@ -216,4 +227,9 @@ public interface AssetRepository extends JpaRepository, Expor @Query(value = "SELECT DISTINCT new org.thingsboard.server.common.data.util.TbPair(a.tenantId , a.type) FROM AssetEntity a") Page> getAllAssetTypes(Pageable pageable); + + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.AssetFields(a.id, a.createdTime, a.tenantId, a.customerId," + + "a.name, a.version, a.type, a.label, a.assetProfileId, a.additionalInfo) FROM AssetEntity a WHERE a.id > :id ORDER BY a.id") + List findAllFields(@Param("id") UUID id, Limit limit); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index 5783ad0c05..c61d894de5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -18,12 +18,15 @@ package org.thingsboard.server.dao.sql.asset; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; +import org.thingsboard.server.common.data.edqs.fields.AssetFields; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -35,12 +38,15 @@ import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.model.sql.AssetEntity; import org.thingsboard.server.dao.model.sql.AssetInfoEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; +import org.thingsboard.server.dao.sql.device.NativeAssetRepository; +import org.thingsboard.server.dao.sql.device.NativeDeviceRepository; import org.thingsboard.server.dao.util.SqlDao; import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; import static org.thingsboard.server.dao.DaoUtil.convertTenantEntityInfosToDto; @@ -55,6 +61,9 @@ public class JpaAssetDao extends JpaAbstractDao implements A @Autowired private AssetRepository assetRepository; + @Autowired + private NativeAssetRepository nativeAssetRepository; + @Autowired private AssetProfileRepository assetProfileRepository; @@ -159,6 +168,16 @@ public class JpaAssetDao extends JpaAbstractDao implements A DaoUtil.toPageable(pageLink, AssetInfoEntity.assetInfoColumnMap))); } + @Override + public PageData findAssetIdsByTenantIdAndAssetProfileId(UUID tenantId, UUID assetProfileId, PageLink pageLink) { + return DaoUtil.pageToPageData(assetRepository.findAssetIdsByTenantIdAndAssetProfileId( + tenantId, + assetProfileId, + pageLink.getTextSearch(), + DaoUtil.toPageable(pageLink))) + .mapData(AssetId::new); + } + @Override public PageData findAssetsByTenantIdAndCustomerIdAndType(UUID tenantId, UUID customerId, String type, PageLink pageLink) { return DaoUtil.toPageData(assetRepository @@ -241,6 +260,12 @@ public class JpaAssetDao extends JpaAbstractDao implements A DaoUtil.toPageable(pageLink, Arrays.asList(new SortOrder("tenantId"), new SortOrder("type"))))); } + @Override + public PageData findProfileEntityIdInfos(PageLink pageLink) { + log.debug("Find profile device id infos by pageLink [{}]", pageLink); + return nativeAssetRepository.findProfileEntityIdInfos(DaoUtil.toPageable(pageLink)); + } + @Override public Long countByTenantId(TenantId tenantId) { return assetRepository.countByTenantId(tenantId.getId()); @@ -267,6 +292,16 @@ public class JpaAssetDao extends JpaAbstractDao implements A .map(AssetId::new).orElse(null); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public List findNextBatch(UUID uuid, int batchSize) { + return assetRepository.findAllFields(uuid, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.ASSET; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java index c64efe01c7..eeab5a338a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.sql.asset; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.PageRequest; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; @@ -23,11 +24,13 @@ import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.asset.AssetProfileInfo; +import org.thingsboard.server.common.data.edqs.fields.AssetProfileFields; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.asset.AssetProfileDao; import org.thingsboard.server.dao.model.sql.AssetProfileEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -37,7 +40,7 @@ import java.util.Optional; import java.util.UUID; @Component -public class JpaAssetProfileDao extends JpaAbstractDao implements AssetProfileDao { +public class JpaAssetProfileDao extends JpaAbstractDao implements AssetProfileDao, TenantEntityDao { @Autowired private AssetProfileRepository assetProfileRepository; @@ -138,6 +141,16 @@ public class JpaAssetProfileDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return assetProfileRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.ASSET_PROFILE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java index f016de2878..e06975a9a5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java @@ -59,4 +59,12 @@ public interface AttributeKvRepository extends JpaRepository findAllKeysByEntityIdsAndAttributeType(@Param("entityIds") List entityIds, @Param("attributeType") int attributeType); + + @Query(value = "SELECT attribute_key, attribute_type, entity_id, bool_v, dbl_v, json_v, last_update_ts, long_v, str_v, version FROM attribute_kv WHERE (entity_id, attribute_type, attribute_key) > " + + "(:entityId, :attributeType, :attributeKey) ORDER BY entity_id, attribute_type, attribute_key LIMIT :batchSize", nativeQuery = true) + List findNextBatch(@Param("entityId") UUID entityId, + @Param("attributeType") int attributeType, + @Param("attributeKey") int attributeKey, + @Param("batchSize") int batchSize); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index 9d477ee5b3..0a8b8f6399 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -49,6 +49,7 @@ import java.util.Collection; import java.util.Comparator; import java.util.List; import java.util.Optional; +import java.util.UUID; import java.util.function.Function; import java.util.stream.Collectors; @@ -152,6 +153,11 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl return DaoUtil.convertDataList(Lists.newArrayList(attributes)); } + @Override + public List findNextBatch(UUID entityId, int attributeType, int attributeKey, int batchSize) { + return attributeKvRepository.findNextBatch(entityId, attributeType, attributeKey, batchSize); + } + @Override public List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId) { if (deviceProfileId != null) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java new file mode 100644 index 0000000000..584a3b5199 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.cf; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; + +import java.util.List; +import java.util.UUID; + +public interface CalculatedFieldLinkRepository extends JpaRepository { + + List findAllByTenantIdAndCalculatedFieldId(UUID tenantId, UUID calculatedFieldId); + + List findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java new file mode 100644 index 0000000000..0f48f3b00d --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.cf; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; + +import java.util.List; +import java.util.UUID; + +public interface CalculatedFieldRepository extends JpaRepository { + + boolean existsByTenantIdAndEntityId(UUID tenantId, UUID entityId); + + List findCalculatedFieldIdsByTenantIdAndEntityId(UUID tenantId, UUID entityId); + + List findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + + Page findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId, Pageable pageable); + + List findAllByTenantId(UUID tenantId); + + List removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + + long countByTenantIdAndEntityId(UUID tenantId, UUID entityId); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java new file mode 100644 index 0000000000..e59ff3f4e6 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -0,0 +1,136 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.cf; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Pageable; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; + +@RequiredArgsConstructor +@Repository +@Slf4j +public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedFieldRepository { + + private final String CF_COUNT_QUERY = "SELECT count(id) FROM calculated_field;"; + private final String CF_QUERY = "SELECT * FROM calculated_field ORDER BY created_time ASC LIMIT %s OFFSET %s"; + + private final String CFL_COUNT_QUERY = "SELECT count(id) FROM calculated_field_link;"; + private final String CFL_QUERY = "SELECT * FROM calculated_field_link ORDER BY created_time ASC LIMIT %s OFFSET %s"; + + private final NamedParameterJdbcTemplate jdbcTemplate; + private final TransactionTemplate transactionTemplate; + + @Override + public PageData findCalculatedFields(Pageable pageable) { + return transactionTemplate.execute(status -> { + long startTs = System.currentTimeMillis(); + int totalElements = jdbcTemplate.queryForObject(CF_COUNT_QUERY, Collections.emptyMap(), Integer.class); + log.debug("Count query took {} ms", System.currentTimeMillis() - startTs); + startTs = System.currentTimeMillis(); + List> rows = jdbcTemplate.queryForList(String.format(CF_QUERY, pageable.getPageSize(), pageable.getOffset()), Collections.emptyMap()); + log.debug("Main query took {} ms", System.currentTimeMillis() - startTs); + int totalPages = pageable.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageable.getPageSize()) : 1; + boolean hasNext = pageable.getPageSize() > 0 && totalElements > pageable.getOffset() + rows.size(); + var data = rows.stream().map(row -> { + + UUID id = (UUID) row.get("id"); + long createdTime = (long) row.get("created_time"); + UUID tenantId = (UUID) row.get("tenant_id"); + EntityType entityType = EntityType.valueOf((String) row.get("entity_type")); + UUID entityId = (UUID) row.get("entity_id"); + CalculatedFieldType type = CalculatedFieldType.valueOf((String) row.get("type")); + String name = (String) row.get("name"); + int configurationVersion = (int) row.get("configuration_version"); + JsonNode configuration = JacksonUtil.toJsonNode((String) row.get("configuration")); + long version = row.get("version") != null ? (long) row.get("version") : 0; + String debugSettings = (String) row.get("debug_settings"); + Object externalIdObj = row.get("external_id"); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setId(new CalculatedFieldId(id)); + calculatedField.setCreatedTime(createdTime); + calculatedField.setTenantId(TenantId.fromUUID(tenantId)); + calculatedField.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedField.setType(type); + calculatedField.setName(name); + calculatedField.setConfigurationVersion(configurationVersion); + calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); + calculatedField.setVersion(version); + calculatedField.setDebugSettings(JacksonUtil.fromString(debugSettings, DebugSettings.class)); + + return calculatedField; + }).collect(Collectors.toList()); + return new PageData<>(data, totalPages, totalElements, hasNext); + }); + } + + @Override + public PageData findCalculatedFieldLinks(Pageable pageable) { + return transactionTemplate.execute(status -> { + long startTs = System.currentTimeMillis(); + int totalElements = jdbcTemplate.queryForObject(CFL_COUNT_QUERY, Collections.emptyMap(), Integer.class); + log.debug("Count query took {} ms", System.currentTimeMillis() - startTs); + startTs = System.currentTimeMillis(); + List> rows = jdbcTemplate.queryForList(String.format(CFL_QUERY, pageable.getPageSize(), pageable.getOffset()), Collections.emptyMap()); + log.debug("Main query took {} ms", System.currentTimeMillis() - startTs); + int totalPages = pageable.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageable.getPageSize()) : 1; + boolean hasNext = pageable.getPageSize() > 0 && totalElements > pageable.getOffset() + rows.size(); + var data = rows.stream().map(row -> { + + UUID id = (UUID) row.get("id"); + long createdTime = (long) row.get("created_time"); + UUID tenantId = (UUID) row.get("tenant_id"); + EntityType entityType = EntityType.valueOf((String) row.get("entity_type")); + UUID entityId = (UUID) row.get("entity_id"); + UUID calculatedFieldId = (UUID) row.get("calculated_field_id"); + JsonNode configuration = JacksonUtil.toJsonNode((String) row.get("configuration")); + + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); + calculatedFieldLink.setId(new CalculatedFieldLinkId(id)); + calculatedFieldLink.setCreatedTime(createdTime); + calculatedFieldLink.setTenantId(new TenantId(tenantId)); + calculatedFieldLink.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedFieldLink.setCalculatedFieldId(new CalculatedFieldId(calculatedFieldId)); + + return calculatedFieldLink; + }).collect(Collectors.toList()); + return new PageData<>(data, totalPages, totalElements, hasNext); + }); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java new file mode 100644 index 0000000000..8922eaca4e --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -0,0 +1,106 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.cf; + +import jakarta.transaction.Transactional; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.cf.CalculatedFieldDao; +import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; +import org.thingsboard.server.dao.sql.JpaAbstractDao; +import org.thingsboard.server.dao.util.SqlDao; + +import java.util.List; +import java.util.UUID; + +@Slf4j +@Component +@AllArgsConstructor +@SqlDao +public class JpaCalculatedFieldDao extends JpaAbstractDao implements CalculatedFieldDao { + + private final CalculatedFieldRepository calculatedFieldRepository; + private final NativeCalculatedFieldRepository nativeCalculatedFieldRepository; + + @Override + public List findAllByTenantId(TenantId tenantId) { + return DaoUtil.convertDataList(calculatedFieldRepository.findAllByTenantId(tenantId.getId())); + } + + @Override + public List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId) { + return calculatedFieldRepository.findCalculatedFieldIdsByTenantIdAndEntityId(tenantId.getId(), entityId.getId()); + } + + @Override + public List findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { + return DaoUtil.convertDataList(calculatedFieldRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId())); + } + + @Override + public List findAll() { + return DaoUtil.convertDataList(calculatedFieldRepository.findAll()); + } + + @Override + public PageData findAll(PageLink pageLink) { + log.debug("Try to find calculated fields by pageLink [{}]", pageLink); + return nativeCalculatedFieldRepository.findCalculatedFields(DaoUtil.toPageable(pageLink)); + } + + @Override + public PageData findAllByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink) { + log.debug("Try to find calculated fields by entityId[{}] and pageLink [{}]", entityId, pageLink); + return DaoUtil.toPageData(calculatedFieldRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId(), DaoUtil.toPageable(pageLink))); + } + + @Override + @Transactional + public List removeAllByEntityId(TenantId tenantId, EntityId entityId) { + return DaoUtil.convertDataList(calculatedFieldRepository.removeAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId())); + } + + @Override + public long countCFByEntityId(TenantId tenantId, EntityId entityId) { + return calculatedFieldRepository.countByTenantIdAndEntityId(tenantId.getId(), entityId.getId()); + } + + @Override + protected Class getEntityClass() { + return CalculatedFieldEntity.class; + } + + @Override + protected JpaRepository getRepository() { + return calculatedFieldRepository; + } + + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java new file mode 100644 index 0000000000..dbb2fd87da --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -0,0 +1,83 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.cf; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.cf.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; +import org.thingsboard.server.dao.sql.JpaAbstractDao; +import org.thingsboard.server.dao.util.SqlDao; + +import java.util.List; +import java.util.UUID; + +@Slf4j +@Component +@AllArgsConstructor +@SqlDao +public class JpaCalculatedFieldLinkDao extends JpaAbstractDao implements CalculatedFieldLinkDao { + + private final CalculatedFieldLinkRepository calculatedFieldLinkRepository; + private final NativeCalculatedFieldRepository nativeCalculatedFieldRepository; + + @Override + public List findCalculatedFieldLinksByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAllByTenantIdAndCalculatedFieldId(tenantId.getId(), calculatedFieldId.getId())); + } + + @Override + public List findCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { + return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId())); + } + + @Override + public List findAll() { + return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAll()); + } + + @Override + public PageData findAll(PageLink pageLink) { + log.debug("Try to find calculated field links by pageLink [{}]", pageLink); + return nativeCalculatedFieldRepository.findCalculatedFieldLinks(DaoUtil.toPageable(pageLink)); + } + + @Override + protected Class getEntityClass() { + return CalculatedFieldLinkEntity.class; + } + + @Override + protected JpaRepository getRepository() { + return calculatedFieldLinkRepository; + } + + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD_LINK; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java new file mode 100644 index 0000000000..f37a5764a0 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.cf; + +import org.springframework.data.domain.Pageable; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.page.PageData; + +public interface NativeCalculatedFieldRepository { + + PageData findCalculatedFields(Pageable pageable); + + PageData findCalculatedFieldLinks(Pageable pageable); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java index af4c96fa63..8ad7311423 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java @@ -15,14 +15,17 @@ */ package org.thingsboard.server.dao.sql.customer; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.CustomerFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.CustomerEntity; +import java.util.List; import java.util.UUID; /** @@ -55,4 +58,8 @@ public interface CustomerRepository extends JpaRepository, nativeQuery = true) Page findCustomersWithTheSameTitle(Pageable pageable); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.CustomerFields(c.id, c.createdTime, c.tenantId, " + + "c.title, c.version, c.additionalInfo, c.country, c.state, c.city, c.address, c.address2, c.zip, c.phone, c.email) " + + "FROM CustomerEntity c WHERE c.id > :id ORDER BY c.id") + List findNextBatch(@Param("id") UUID id, Limit limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index 4c3d0083a6..75e7179391 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -16,10 +16,12 @@ package org.thingsboard.server.dao.sql.customer; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.CustomerFields; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -30,6 +32,7 @@ import org.thingsboard.server.dao.model.sql.CustomerEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; +import java.util.List; import java.util.Optional; import java.util.UUID; @@ -104,6 +107,16 @@ public class JpaCustomerDao extends JpaAbstractDao imp ); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return customerRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.CUSTOMER; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardInfoRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardInfoRepository.java index cddbe3e347..7624ddc738 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardInfoRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardInfoRepository.java @@ -78,13 +78,21 @@ public interface DashboardInfoRepository extends JpaRepository findByTenantAndImageLink(@Param("tenantId") UUID tenantId, @Param("imageLink") String imageLink, @Param("lmt") int lmt); + List findByTenantAndImageLink(@Param("tenantId") UUID tenantId, @Param("imageLink") String imageLink, @Param("limit") int limit); @Query(nativeQuery = true, - value = "SELECT * FROM dashboard d WHERE d.image = :imageLink or d.configuration ILIKE CONCAT('%\"', :imageLink, '\"%') limit :lmt" + value = "SELECT * FROM dashboard d WHERE d.image = :imageLink or d.configuration ILIKE CONCAT('%\"', :imageLink, '\"%') limit :limit" ) - List findByImageLink(@Param("imageLink") String imageLink, @Param("lmt") int lmt); + List findByImageLink(@Param("imageLink") String imageLink, @Param("limit") int limit); + + @Query(value = "SELECT * FROM dashboard d WHERE d.tenant_id = :tenantId and d.configuration ILIKE CONCAT('%', :link, '%') limit :limit", + nativeQuery = true) + List findDashboardInfosByTenantIdAndResourceLink(@Param("tenantId") UUID tenantId, @Param("link") String link, @Param("limit") int limit); + + @Query(value = "SELECT * FROM dashboard d WHERE d.configuration ILIKE CONCAT('%', :link, '%') limit :limit", + nativeQuery = true) + List findDashboardInfosByResourceLink(@Param("link") String link, @Param("limit") int limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java index 94b6cd541c..f4e934e64b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.dao.sql.dashboard; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.DashboardFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.DashboardEntity; @@ -46,4 +48,7 @@ public interface DashboardRepository extends JpaRepository findAllIds(Pageable pageable); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.DashboardFields(d.id, d.createdTime, d.tenantId, " + + "d.assignedCustomers, d.title, d.version) FROM DashboardEntity d WHERE d.id > :id ORDER BY d.id") + List findNextBatch(@Param("id") UUID id, Limit limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java index ea78114ead..2d796c6917 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java @@ -16,10 +16,12 @@ package org.thingsboard.server.dao.sql.dashboard; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.DashboardFields; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -90,6 +92,16 @@ public class JpaDashboardDao extends JpaAbstractDao return DaoUtil.pageToPageData(dashboardRepository.findAllIds(DaoUtil.toPageable(pageLink)).map(DashboardId::new)); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return dashboardRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.DASHBOARD; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java index 249e9d6e5b..bc07139725 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java @@ -133,4 +133,15 @@ public class JpaDashboardInfoDao extends JpaAbstractDao findByImageLink(String imageLink, int limit) { return DaoUtil.convertDataList(dashboardInfoRepository.findByImageLink(imageLink, limit)); } + + @Override + public List findByTenantIdAndResourceLink(TenantId tenantId, String url, int limit) { + return DaoUtil.convertDataList(dashboardInfoRepository.findDashboardInfosByTenantIdAndResourceLink(tenantId.getId(), url, limit)); + } + + @Override + public List findByResourceLink(String link, int limit) { + return DaoUtil.convertDataList(dashboardInfoRepository.findDashboardInfosByResourceLink(link, limit)); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/AbstractNativeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/AbstractNativeRepository.java new file mode 100644 index 0000000000..bba84503f4 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/AbstractNativeRepository.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.device; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Pageable; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.server.common.data.page.PageData; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +@RequiredArgsConstructor +@Repository +@Slf4j +public class AbstractNativeRepository { + + private final NamedParameterJdbcTemplate jdbcTemplate; + private final TransactionTemplate transactionTemplate; + + protected PageData find(String countQuery, String findQuery, Pageable pageable, Function, T> mapper) { + return transactionTemplate.execute(status -> { + long startTs = System.currentTimeMillis(); + int totalElements = jdbcTemplate.queryForObject(countQuery, Collections.emptyMap(), Integer.class); + log.debug("Count query took {} ms", System.currentTimeMillis() - startTs); + startTs = System.currentTimeMillis(); + List> rows = jdbcTemplate.queryForList(String.format(findQuery, pageable.getPageSize(), pageable.getOffset()), Collections.emptyMap()); + log.debug("Main query took {} ms", System.currentTimeMillis() - startTs); + int totalPages = pageable.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageable.getPageSize()) : 1; + boolean hasNext = pageable.getPageSize() > 0 && totalElements > pageable.getOffset() + rows.size(); + var data = rows.stream().map(mapper).collect(Collectors.toList()); + return new PageData<>(data, totalPages, totalElements, hasNext); + }); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java new file mode 100644 index 0000000000..43d66e2ff0 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.device; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Pageable; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.server.common.data.DeviceIdInfo; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; + +import java.util.UUID; + +@Repository +@Slf4j +public class DefaultNativeAssetRepository extends AbstractNativeRepository implements NativeAssetRepository { + + private final String COUNT_QUERY = "SELECT count(id) FROM asset;"; + + public DefaultNativeAssetRepository(NamedParameterJdbcTemplate jdbcTemplate, TransactionTemplate transactionTemplate) { + super(jdbcTemplate, transactionTemplate); + } + + @Override + public PageData findProfileEntityIdInfos(Pageable pageable) { + String PROFILE_DEVICE_ID_INFO_QUERY = "SELECT tenant_id as tenantId, asset_profile_id as profileId, id as id FROM asset ORDER BY created_time ASC LIMIT %s OFFSET %s"; + return find(COUNT_QUERY, PROFILE_DEVICE_ID_INFO_QUERY, pageable, row -> { + AssetId id = new AssetId((UUID) row.get("id")); + AssetProfileId profileId = new AssetProfileId((UUID) row.get("profileId")); + var tenantIdObj = row.get("tenantId"); + return ProfileEntityIdInfo.create(tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(), profileId, id); + }); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java index 4556ac555e..776dedc2d5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java @@ -15,50 +15,50 @@ */ package org.thingsboard.server.dao.sql.device; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.domain.Pageable; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.stereotype.Repository; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.server.common.data.DeviceIdInfo; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; -import java.util.Collections; -import java.util.List; -import java.util.Map; import java.util.UUID; -import java.util.stream.Collectors; -@RequiredArgsConstructor @Repository @Slf4j -public class DefaultNativeDeviceRepository implements NativeDeviceRepository { +public class DefaultNativeDeviceRepository extends AbstractNativeRepository implements NativeDeviceRepository { private final String COUNT_QUERY = "SELECT count(id) FROM device;"; - private final String QUERY = "SELECT tenant_id as tenantId, customer_id as customerId, id as id FROM device ORDER BY created_time ASC LIMIT %s OFFSET %s"; - private final NamedParameterJdbcTemplate jdbcTemplate; - private final TransactionTemplate transactionTemplate; + + public DefaultNativeDeviceRepository(NamedParameterJdbcTemplate jdbcTemplate, TransactionTemplate transactionTemplate) { + super(jdbcTemplate, transactionTemplate); + } @Override public PageData findDeviceIdInfos(Pageable pageable) { - return transactionTemplate.execute(status -> { - long startTs = System.currentTimeMillis(); - int totalElements = jdbcTemplate.queryForObject(COUNT_QUERY, Collections.emptyMap(), Integer.class); - log.debug("Count query took {} ms", System.currentTimeMillis() - startTs); - startTs = System.currentTimeMillis(); - List> rows = jdbcTemplate.queryForList(String.format(QUERY, pageable.getPageSize(), pageable.getOffset()), Collections.emptyMap()); - log.debug("Main query took {} ms", System.currentTimeMillis() - startTs); - int totalPages = pageable.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageable.getPageSize()) : 1; - boolean hasNext = pageable.getPageSize() > 0 && totalElements > pageable.getOffset() + rows.size(); - var data = rows.stream().map(row -> { - UUID id = (UUID) row.get("id"); - var tenantIdObj = row.get("tenantId"); - var customerIdObj = row.get("customerId"); - return new DeviceIdInfo(tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(), customerIdObj != null ? (UUID) customerIdObj : null, id); - }).collect(Collectors.toList()); - return new PageData<>(data, totalPages, totalElements, hasNext); + String DEVICE_ID_INFO_QUERY = "SELECT tenant_id as tenantId, customer_id as customerId, id as id FROM device ORDER BY created_time ASC LIMIT %s OFFSET %s"; + return find(COUNT_QUERY, DEVICE_ID_INFO_QUERY, pageable, row -> { + UUID id = (UUID) row.get("id"); + var tenantIdObj = row.get("tenantId"); + var customerIdObj = row.get("customerId"); + return new DeviceIdInfo(tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(), customerIdObj != null ? (UUID) customerIdObj : null, id); }); } + + @Override + public PageData findProfileEntityIdInfos(Pageable pageable) { + String PROFILE_DEVICE_ID_INFO_QUERY = "SELECT tenant_id as tenantId, device_profile_id as profileId, id as id FROM device ORDER BY created_time ASC LIMIT %s OFFSET %s"; + return find(COUNT_QUERY, PROFILE_DEVICE_ID_INFO_QUERY, pageable, row -> { + DeviceId id = new DeviceId((UUID) row.get("id")); + DeviceProfileId profileId = new DeviceProfileId((UUID) row.get("profileId")); + var tenantIdObj = row.get("tenantId"); + return ProfileEntityIdInfo.create(tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(), profileId, id); + }); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java index 2f84d24420..7b938ea274 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.device; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; @@ -36,4 +38,8 @@ public interface DeviceCredentialsRepository extends JpaRepository findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java index 34dac8c7a4..88d4780f9d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.sql.device; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; @@ -23,6 +24,7 @@ import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityInfo; +import org.thingsboard.server.common.data.edqs.fields.DeviceProfileFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.DeviceProfileEntity; @@ -92,4 +94,7 @@ public interface DeviceProfileRepository extends JpaRepository findAllTenantDeviceProfileNames(@Param("tenantId") UUID tenantId); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.DeviceProfileFields(d.id, d.createdTime, d.tenantId," + + "d.name, d.version, d.type, d.isDefault) FROM DeviceProfileEntity d WHERE d.id > :id ORDER BY d.id") + List findNextBatch(@Param("id") UUID id, Limit limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java index 86048a9014..f4c2fed9fa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java @@ -15,12 +15,14 @@ */ package org.thingsboard.server.dao.sql.device; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.edqs.fields.DeviceFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.DeviceEntity; import org.thingsboard.server.dao.model.sql.DeviceInfoEntity; @@ -81,6 +83,14 @@ public interface DeviceRepository extends JpaRepository, Exp @Param("textSearch") String textSearch, Pageable pageable); + @Query("SELECT d.id FROM DeviceEntity d WHERE d.tenantId = :tenantId " + + "AND d.deviceProfileId = :deviceProfileId " + + "AND (:textSearch IS NULL OR ilike(d.type, CONCAT('%', :textSearch, '%')) = true)") + Page findIdsByTenantIdAndDeviceProfileId(@Param("tenantId") UUID tenantId, + @Param("deviceProfileId") UUID deviceProfileId, + @Param("textSearch") String textSearch, + Pageable pageable); + @Query("SELECT d FROM DeviceEntity d WHERE d.tenantId = :tenantId " + "AND d.deviceProfileId = :deviceProfileId " + "AND d.firmwareId IS NULL") @@ -194,4 +204,9 @@ public interface DeviceRepository extends JpaRepository, Exp @Query("SELECT externalId FROM DeviceEntity WHERE id = :id") UUID getExternalIdById(@Param("id") UUID id); + + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.DeviceFields(d.id, d.createdTime, d.tenantId, d.customerId," + + "d.name, d.version, d.type, d.label, d.deviceProfileId, d.additionalInfo) FROM DeviceEntity d WHERE d.id > :id ORDER BY d.id") + List findNextBatch(@Param("id") UUID id, Limit limit); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java index 6d9093dc0b..7445d058d4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java @@ -21,8 +21,11 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.device.DeviceCredentialsDao; import org.thingsboard.server.dao.model.sql.DeviceCredentialsEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -36,7 +39,7 @@ import java.util.UUID; @Slf4j @Component @SqlDao -public class JpaDeviceCredentialsDao extends JpaAbstractDao implements DeviceCredentialsDao { +public class JpaDeviceCredentialsDao extends JpaAbstractDao implements DeviceCredentialsDao, TenantEntityDao { @Autowired private DeviceCredentialsRepository deviceCredentialsRepository; @@ -67,4 +70,9 @@ public class JpaDeviceCredentialsDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(deviceCredentialsRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index ece841de6c..34835f52f1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.device; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; @@ -29,7 +30,9 @@ import org.thingsboard.server.common.data.DeviceInfoFilter; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.fields.DeviceFields; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; @@ -173,6 +176,17 @@ public class JpaDeviceDao extends JpaAbstractDao implement DaoUtil.toPageable(pageLink))); } + @Override + public PageData findDeviceIdsByTenantIdAndDeviceProfileId(UUID tenantId, UUID deviceProfileId, PageLink pageLink) { + return DaoUtil.pageToPageData( + deviceRepository.findIdsByTenantIdAndDeviceProfileId( + tenantId, + deviceProfileId, + pageLink.getTextSearch(), + DaoUtil.toPageable(pageLink))) + .mapData(DeviceId::new); + } + @Override public PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(UUID tenantId, UUID deviceProfileId, @@ -261,6 +275,12 @@ public class JpaDeviceDao extends JpaAbstractDao implement return nativeDeviceRepository.findDeviceIdInfos(DaoUtil.toPageable(pageLink)); } + @Override + public PageData findProfileEntityIdInfos(PageLink pageLink) { + log.debug("Find profile device id infos by pageLink [{}]", pageLink); + return nativeDeviceRepository.findProfileEntityIdInfos(DaoUtil.toPageable(pageLink)); + } + @Override public Device findByTenantIdAndExternalId(UUID tenantId, UUID externalId) { return DaoUtil.getData(deviceRepository.findByTenantIdAndExternalId(tenantId, externalId)); @@ -282,6 +302,16 @@ public class JpaDeviceDao extends JpaAbstractDao implement .map(DeviceId::new).orElse(null); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return deviceRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.DEVICE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java index d68033d14b..ebd8c78ed0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.sql.device; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.PageRequest; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; @@ -25,11 +26,13 @@ import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.fields.DeviceProfileFields; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.device.DeviceProfileDao; import org.thingsboard.server.dao.model.sql.DeviceProfileEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -41,7 +44,7 @@ import java.util.UUID; @Component @SqlDao -public class JpaDeviceProfileDao extends JpaAbstractDao implements DeviceProfileDao { +public class JpaDeviceProfileDao extends JpaAbstractDao implements DeviceProfileDao, TenantEntityDao { @Autowired private DeviceProfileRepository deviceProfileRepository; @@ -156,6 +159,16 @@ public class JpaDeviceProfileDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findDeviceProfiles(tenantId, pageLink); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return deviceProfileRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.DEVICE_PROFILE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeAssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeAssetRepository.java new file mode 100644 index 0000000000..42f6b4c819 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeAssetRepository.java @@ -0,0 +1,18 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.device; + +public interface NativeAssetRepository extends NativeProfileEntityRepository {} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeDeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeDeviceRepository.java index 53f70ed046..b9fcf75ba4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeDeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeDeviceRepository.java @@ -17,9 +17,10 @@ package org.thingsboard.server.dao.sql.device; import org.springframework.data.domain.Pageable; import org.thingsboard.server.common.data.DeviceIdInfo; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.page.PageData; -public interface NativeDeviceRepository { +public interface NativeDeviceRepository extends NativeProfileEntityRepository { PageData findDeviceIdInfos(Pageable pageable); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeProfileEntityRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeProfileEntityRepository.java new file mode 100644 index 0000000000..750f0c8787 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeProfileEntityRepository.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.device; + +import org.springframework.data.domain.Pageable; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; +import org.thingsboard.server.common.data.page.PageData; + +public interface NativeProfileEntityRepository { + + PageData findProfileEntityIdInfos(Pageable pageable); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeRepository.java index 245b47ae63..c11db0a348 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeRepository.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.dao.sql.edge; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.EdgeFields; import org.thingsboard.server.dao.model.sql.EdgeEntity; import org.thingsboard.server.dao.model.sql.EdgeInfoEntity; @@ -154,4 +156,7 @@ public interface EdgeRepository extends JpaRepository { EdgeEntity findByRoutingKey(String routingKey); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.EdgeFields(e.id, e.createdTime, e.tenantId, e.customerId," + + "e.name, e.version, e.type, e.label, e.additionalInfo) FROM EdgeEntity e WHERE e.id > :id ORDER BY e.id") + List findNextBatch(@Param("id") UUID id, Limit limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java index b1c51c73c1..3f45b1ca1a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java @@ -18,12 +18,14 @@ package org.thingsboard.server.dao.sql.edge; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeInfo; +import org.thingsboard.server.common.data.edqs.fields.EdgeFields; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -219,6 +221,16 @@ public class JpaEdgeDao extends JpaAbstractDao implements Edge return edgeRepository.countByTenantId(tenantId.getId()); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findEdgesByTenantId(tenantId.getId(), pageLink); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return edgeRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.EDGE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java index 0112be4967..6094e9b171 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.dao.sql.entityview; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.EntityViewFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.EntityViewEntity; import org.thingsboard.server.dao.model.sql.EntityViewInfoEntity; @@ -145,4 +147,8 @@ public interface EntityViewRepository extends JpaRepository :id ORDER BY e.id") + List findNextBatch(@Param("id") UUID id, Limit limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java index a6c4457f4f..44d8a09ff4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java @@ -18,17 +18,20 @@ package org.thingsboard.server.dao.sql.entityview; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EntityViewInfo; +import org.thingsboard.server.common.data.edqs.fields.EntityViewFields; import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.entityview.EntityViewDao; import org.thingsboard.server.dao.model.sql.EntityViewEntity; import org.thingsboard.server.dao.model.sql.EntityViewInfoEntity; @@ -47,8 +50,7 @@ import static org.thingsboard.server.dao.DaoUtil.convertTenantEntityTypesToDto; @Component @Slf4j @SqlDao -public class JpaEntityViewDao extends JpaAbstractDao - implements EntityViewDao { +public class JpaEntityViewDao extends JpaAbstractDao implements EntityViewDao, TenantEntityDao { @Autowired private EntityViewRepository entityViewRepository; @@ -218,8 +220,19 @@ public class JpaEntityViewDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return entityViewRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.ENTITY_VIEW; } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java new file mode 100644 index 0000000000..482905d362 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java @@ -0,0 +1,145 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.event; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; +import org.thingsboard.server.dao.model.sql.CalculatedFieldDebugEventEntity; + +import java.util.List; +import java.util.UUID; + +public interface CalculatedFieldDebugEventRepository extends EventRepository, JpaRepository { + + @Override + @Query(nativeQuery = true, value = "SELECT * FROM cf_debug_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit") + List findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit); + + @Override + @Query("SELECT e FROM CalculatedFieldDebugEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + Pageable pageable); + + @Query(nativeQuery = true, + value = "SELECT * FROM cf_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:calculatedFieldId IS NULL OR e.cf_id = uuid(:calculatedFieldId)) " + + "AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " + + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + + "AND (:eventArguments IS NULL OR e.e_args ILIKE concat('%', :eventArguments, '%')) " + + "AND (:eventResult IS NULL OR e.e_result ILIKE concat('%', :eventResult, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + , + countQuery = "SELECT count(*) FROM cf_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:calculatedFieldId IS NULL OR e.cf_id = uuid(:calculatedFieldId)) " + + "AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " + + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + + "AND (:eventArguments IS NULL OR e.e_args ILIKE concat('%', :eventArguments, '%')) " + + "AND (:eventResult IS NULL OR e.e_result ILIKE concat('%', :eventResult, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String serviceId, + @Param("calculatedFieldId") UUID calculatedFieldId, + @Param("eventEntityId") String eventEntityId, + @Param("eventEntityType") String eventEntityType, + @Param("msgId") String eventMsgId, + @Param("msgType") String eventMsgType, + @Param("eventArguments") String eventArguments, + @Param("eventResult") String eventResult, + @Param("isError") boolean isError, + @Param("error") String error, + Pageable pageable); + + @Transactional + @Modifying + @Query("DELETE FROM CalculatedFieldDebugEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime); + + @Transactional + @Modifying + @Query(nativeQuery = true, + value = "DELETE FROM cf_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:calculatedFieldId IS NULL OR e.cf_id = uuid(:calculatedFieldId)) " + + "AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " + + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + + "AND (:eventArguments IS NULL OR e.e_args ILIKE concat('%', :eventArguments, '%')) " + + "AND (:eventResult IS NULL OR e.e_result ILIKE concat('%', :eventResult, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))") + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String serviceId, + @Param("calculatedFieldId") UUID calculatedFieldId, + @Param("eventEntityId") String eventEntityId, + @Param("eventEntityType") String eventEntityType, + @Param("msgId") String eventMsgId, + @Param("msgType") String eventMsgType, + @Param("eventArguments") String eventArguments, + @Param("eventResult") String eventResult, + @Param("isError") boolean isError, + @Param("error") String error); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/DedicatedJpaEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/DedicatedJpaEventDao.java index abe95fb8f4..e13dfacf81 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/DedicatedJpaEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/DedicatedJpaEventDao.java @@ -36,10 +36,11 @@ public class DedicatedJpaEventDao extends JpaBaseEventDao { RuleNodeDebugEventRepository ruleNodeDebugEventRepository, RuleChainDebugEventRepository ruleChainDebugEventRepository, ScheduledLogExecutorComponent logExecutor, - StatsFactory statsFactory) { + StatsFactory statsFactory, + CalculatedFieldDebugEventRepository cfDebugEventRepository) { super(partitionConfiguration, partitioningRepository, lcEventRepository, statsEventRepository, errorEventRepository, eventInsertRepository, ruleNodeDebugEventRepository, - ruleChainDebugEventRepository, logExecutor, statsFactory); + ruleChainDebugEventRepository, logExecutor, statsFactory, cfDebugEventRepository); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java index cd25577214..142924eac6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java @@ -25,6 +25,7 @@ import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.support.TransactionCallbackWithoutResult; import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventType; @@ -81,6 +82,9 @@ public class EventInsertRepository { insertStmtMap.put(EventType.DEBUG_RULE_CHAIN, "INSERT INTO " + EventType.DEBUG_RULE_CHAIN.getTable() + " (id, tenant_id, ts, entity_id, service_id, e_message, e_error) " + "VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); + insertStmtMap.put(EventType.DEBUG_CALCULATED_FIELD, "INSERT INTO " + EventType.DEBUG_CALCULATED_FIELD.getTable() + + " (id, tenant_id, ts, entity_id, service_id, cf_id, e_entity_id, e_entity_type, e_msg_id, e_msg_type, e_args, e_result, e_error) " + + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); } public void save(List entities) { @@ -107,6 +111,8 @@ public class EventInsertRepository { return getRuleNodeEventSetter(events); case DEBUG_RULE_CHAIN: return getRuleChainEventSetter(events); + case DEBUG_CALCULATED_FIELD: + return getCalculatedFieldEventSetter(events); default: throw new RuntimeException(eventType + " support is not implemented!"); } @@ -206,6 +212,29 @@ public class EventInsertRepository { }; } + private BatchPreparedStatementSetter getCalculatedFieldEventSetter(List events) { + return new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + CalculatedFieldDebugEvent event = (CalculatedFieldDebugEvent) events.get(i); + setCommonEventFields(ps, event); + safePutUUID(ps, 6, event.getCalculatedFieldId().getId()); + safePutUUID(ps, 7, event.getEventEntity() != null ? event.getEventEntity().getId() : null); + safePutString(ps, 8, event.getEventEntity() != null ? event.getEventEntity().getEntityType().name() : null); + safePutUUID(ps, 9, event.getMsgId()); + safePutString(ps, 10, event.getMsgType()); + safePutString(ps, 11, event.getArguments()); + safePutString(ps, 12, event.getResult()); + safePutString(ps, 13, event.getError()); + } + + @Override + public int getBatchSize() { + return events.size(); + } + }; + } + void safePutString(PreparedStatement ps, int parameterIdx, String value) throws SQLException { if (value != null) { ps.setString(parameterIdx, replaceNullChars(value)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java index 3945c80c43..592ed21f9a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java @@ -24,6 +24,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEventFilter; import org.thingsboard.server.common.data.event.ErrorEventFilter; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; @@ -72,6 +73,7 @@ public class JpaBaseEventDao implements EventDao { private final RuleChainDebugEventRepository ruleChainDebugEventRepository; private final ScheduledLogExecutorComponent logExecutor; private final StatsFactory statsFactory; + private final CalculatedFieldDebugEventRepository calculatedFieldDebugEventRepository; @Value("${sql.events.batch_size:10000}") private int batchSize; @@ -110,6 +112,7 @@ public class JpaBaseEventDao implements EventDao { repositories.put(EventType.ERROR, errorEventRepository); repositories.put(EventType.DEBUG_RULE_NODE, ruleNodeDebugEventRepository); repositories.put(EventType.DEBUG_RULE_CHAIN, ruleChainDebugEventRepository); + repositories.put(EventType.DEBUG_CALCULATED_FIELD, calculatedFieldDebugEventRepository); } @PreDestroy @@ -158,6 +161,8 @@ public class JpaBaseEventDao implements EventDao { return findEventByFilter(tenantId, entityId, (ErrorEventFilter) eventFilter, pageLink); case STATS: return findEventByFilter(tenantId, entityId, (StatisticsEventFilter) eventFilter, pageLink); + case DEBUG_CALCULATED_FIELD: + return findEventByFilter(tenantId, entityId, (CalculatedFieldDebugEventFilter) eventFilter, pageLink); default: throw new RuntimeException("Not supported event type: " + eventFilter.getEventType()); } @@ -193,6 +198,8 @@ public class JpaBaseEventDao implements EventDao { case STATS: removeEventsByFilter(tenantId, entityId, (StatisticsEventFilter) eventFilter, startTime, endTime); break; + case DEBUG_CALCULATED_FIELD: + removeEventsByFilter(tenantId, entityId, (CalculatedFieldDebugEventFilter) eventFilter, startTime, endTime); default: throw new RuntimeException("Not supported event type: " + eventFilter.getEventType()); } @@ -286,6 +293,28 @@ public class JpaBaseEventDao implements EventDao { ); } + private PageData findEventByFilter(UUID tenantId, UUID entityId, CalculatedFieldDebugEventFilter eventFilter, TimePageLink pageLink) { + parseUUID(eventFilter.getEntityId(), "Entity Id"); + parseUUID(eventFilter.getMsgId(), "Message Id"); + return DaoUtil.toPageData( + calculatedFieldDebugEventRepository.findEvents( + tenantId, + entityId, + pageLink.getStartTime(), + pageLink.getEndTime(), + eventFilter.getServer(), + entityId, + eventFilter.getEntityId(), + eventFilter.getEntityType(), + eventFilter.getMsgId(), + eventFilter.getMsgType(), + eventFilter.getArguments(), + eventFilter.getResult(), + eventFilter.isError(), + eventFilter.getErrorStr(), + DaoUtil.toPageable(pageLink, EventEntity.eventColumnMap))); + } + private void removeEventsByFilter(UUID tenantId, UUID entityId, RuleChainDebugEventFilter eventFilter, Long startTime, Long endTime) { ruleChainDebugEventRepository.removeEvents( tenantId, @@ -360,6 +389,26 @@ public class JpaBaseEventDao implements EventDao { ); } + private void removeEventsByFilter(UUID tenantId, UUID entityId, CalculatedFieldDebugEventFilter eventFilter, Long startTime, Long endTime) { + parseUUID(eventFilter.getEntityId(), "Entity Id"); + parseUUID(eventFilter.getMsgId(), "Message Id"); + calculatedFieldDebugEventRepository.removeEvents( + tenantId, + entityId, + startTime, + endTime, + eventFilter.getServer(), + entityId, + eventFilter.getEntityId(), + eventFilter.getEntityType(), + eventFilter.getMsgId(), + eventFilter.getMsgType(), + eventFilter.getArguments(), + eventFilter.getResult(), + eventFilter.isError(), + eventFilter.getErrorStr()); + } + @Override public List findLatestEvents(UUID tenantId, UUID entityId, EventType eventType, int limit) { return DaoUtil.convertDataList(getEventRepository(eventType).findLatestEvents(tenantId, entityId, limit)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRuleDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRuleDao.java index 6781dda593..739423ee70 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRuleDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRuleDao.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.notification.rule.trigger.config.Notif import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.NotificationRuleEntity; import org.thingsboard.server.dao.model.sql.NotificationRuleInfoEntity; import org.thingsboard.server.dao.notification.NotificationRuleDao; @@ -41,7 +42,7 @@ import java.util.UUID; @Component @SqlDao @RequiredArgsConstructor -public class JpaNotificationRuleDao extends JpaAbstractDao implements NotificationRuleDao { +public class JpaNotificationRuleDao extends JpaAbstractDao implements NotificationRuleDao, TenantEntityDao { private final NotificationRuleRepository notificationRuleRepository; @@ -101,6 +102,11 @@ public class JpaNotificationRuleDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + @Override protected Class getEntityClass() { return NotificationRuleEntity.class; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTargetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTargetDao.java index 07799d2388..1efce2de01 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTargetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTargetDao.java @@ -101,6 +101,11 @@ public class JpaNotificationTargetDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + @Override protected Class getEntityClass() { return NotificationTargetEntity.class; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTemplateDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTemplateDao.java index 8941bf119d..0f7d6e1d64 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTemplateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTemplateDao.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.notification.template.NotificationTemp import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.NotificationTemplateEntity; import org.thingsboard.server.dao.notification.NotificationTemplateDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -38,7 +39,7 @@ import java.util.UUID; @Component @SqlDao @RequiredArgsConstructor -public class JpaNotificationTemplateDao extends JpaAbstractDao implements NotificationTemplateDao { +public class JpaNotificationTemplateDao extends JpaAbstractDao implements NotificationTemplateDao, TenantEntityDao { private final NotificationTemplateRepository notificationTemplateRepository; @@ -83,6 +84,11 @@ public class JpaNotificationTemplateDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + @Override protected JpaRepository getRepository() { return notificationTemplateRepository; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java index 24ba852cd8..780f67932e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java @@ -19,9 +19,14 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.OtaPackageEntity; import org.thingsboard.server.dao.ota.OtaPackageDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -32,7 +37,7 @@ import java.util.UUID; @Slf4j @Component @SqlDao -public class JpaOtaPackageDao extends JpaAbstractDao implements OtaPackageDao { +public class JpaOtaPackageDao extends JpaAbstractDao implements OtaPackageDao, TenantEntityDao { @Autowired private OtaPackageRepository otaPackageRepository; @@ -52,6 +57,12 @@ public class JpaOtaPackageDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(otaPackageRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.OTA_PACKAGE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageInfoDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageInfoDao.java index dec6d36ec6..208b296d1c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageInfoDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageInfoDao.java @@ -93,4 +93,5 @@ public class JpaOtaPackageInfoDao extends JpaAbstractDao { + @Query(value = "SELECT COALESCE(SUM(ota.data_size), 0) FROM ota_package ota WHERE ota.tenant_id = :tenantId AND ota.data IS NOT NULL", nativeQuery = true) Long sumDataSizeByTenantId(@Param("tenantId") UUID tenantId); + + Page findByTenantId(UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/AlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/AlarmQueryRepository.java index adbb3fc200..4380186786 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/AlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/AlarmQueryRepository.java @@ -30,6 +30,6 @@ public interface AlarmQueryRepository { PageData findAlarmDataByQueryForEntities(TenantId tenantId, AlarmDataQuery query, Collection orderedEntityIds); - long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query); + long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query, Collection orderedEntityIds); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java index 957e52142a..9e20a54b14 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.QueryContext; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataPageLink; @@ -128,7 +129,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { public PageData findAlarmDataByQueryForEntities(TenantId tenantId, AlarmDataQuery query, Collection orderedEntityIds) { return transactionTemplate.execute(trStatus -> { AlarmDataPageLink pageLink = query.getPageLink(); - QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, null, EntityType.ALARM)); + SqlQueryContext ctx = new SqlQueryContext(new QueryContext(tenantId, null, EntityType.ALARM)); ctx.addUuidListParameter("entity_ids", orderedEntityIds.stream().map(EntityId::getId).collect(Collectors.toList())); StringBuilder selectPart = new StringBuilder(FIELDS_SELECTION); StringBuilder fromPart = new StringBuilder(" from alarm_info a "); @@ -314,25 +315,41 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { } @Override - public long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query) { - QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, null, EntityType.ALARM)); + public long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query, Collection orderedEntityIds) { + SqlQueryContext ctx = new SqlQueryContext(new QueryContext(tenantId, null, EntityType.ALARM)); if (query.isSearchPropagatedAlarms()) { ctx.append("select count(distinct(a.id)) from alarm_info a "); ctx.append(JOIN_ENTITY_ALARMS); - ctx.append("where a.tenant_id = :tenantId and ea.tenant_id = :tenantId"); - ctx.addUuidParameter("tenantId", tenantId.getId()); - if (customerId != null && !customerId.isNullUid()) { - ctx.append(" and a.customer_id = :customerId and ea.customer_id = :customerId"); - ctx.addUuidParameter("customerId", customerId.getId()); + if (orderedEntityIds != null) { + if (orderedEntityIds.isEmpty()) { + return 0; + } + ctx.addUuidListParameter("entity_filter_entity_ids", orderedEntityIds.stream().map(EntityId::getId).collect(Collectors.toList())); + ctx.append("where ea.entity_id in (:entity_filter_entity_ids)"); + } else { + ctx.append("where a.tenant_id = :tenantId and ea.tenant_id = :tenantId"); + ctx.addUuidParameter("tenantId", tenantId.getId()); + if (customerId != null && !customerId.isNullUid()) { + ctx.append(" and a.customer_id = :customerId and ea.customer_id = :customerId"); + ctx.addUuidParameter("customerId", customerId.getId()); + } } } else { ctx.append("select count(id) from alarm_info a "); - ctx.append("where a.tenant_id = :tenantId"); - ctx.addUuidParameter("tenantId", tenantId.getId()); - if (customerId != null && !customerId.isNullUid()) { - ctx.append(" and a.customer_id = :customerId"); - ctx.addUuidParameter("customerId", customerId.getId()); + if (orderedEntityIds != null) { + if (orderedEntityIds.isEmpty()) { + return 0; + } + ctx.addUuidListParameter("entity_filter_entity_ids", orderedEntityIds.stream().map(EntityId::getId).collect(Collectors.toList())); + ctx.append("where a.originator_id in (:entity_filter_entity_ids)"); + } else { + ctx.append("where a.tenant_id = :tenantId"); + ctx.addUuidParameter("tenantId", tenantId.getId()); + if (customerId != null && !customerId.isNullUid()) { + ctx.append(" and a.customer_id = :customerId"); + ctx.addUuidParameter("customerId", customerId.getId()); + } } } @@ -402,7 +419,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { }); } - private String buildTextSearchQuery(QueryContext ctx, List selectionMapping, String searchText) { + private String buildTextSearchQuery(SqlQueryContext ctx, List selectionMapping, String searchText) { if (!StringUtils.isEmpty(searchText) && selectionMapping != null && !selectionMapping.isEmpty()) { String lowerSearchText = searchText.toLowerCase() + "%"; List searchPredicates = selectionMapping.stream() @@ -420,7 +437,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { } } - private String buildPermissionsQuery(TenantId tenantId, QueryContext ctx) { + private String buildPermissionsQuery(TenantId tenantId, SqlQueryContext ctx) { StringBuilder permissionsQuery = new StringBuilder(); ctx.addUuidParameter("permissions_tenant_id", tenantId.getId()); permissionsQuery.append(" a.tenant_id = :permissions_tenant_id and ea.tenant_id = :permissions_tenant_id "); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index ef712fa286..8f3fe71f35 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.QueryContext; import org.thingsboard.server.common.data.query.ApiUsageStateFilter; import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; import org.thingsboard.server.common.data.query.AssetTypeFilter; @@ -334,7 +335,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { @Override public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query) { EntityType entityType = resolveEntityType(query.getEntityFilter()); - QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, customerId, entityType, TenantId.SYS_TENANT_ID.equals(tenantId))); + SqlQueryContext ctx = new SqlQueryContext(new QueryContext(tenantId, customerId, entityType, TenantId.SYS_TENANT_ID.equals(tenantId))); if (query.getKeyFilters() == null || query.getKeyFilters().isEmpty()) { ctx.append("select count(e.id) from "); ctx.append(addEntityTableQuery(ctx, query.getEntityFilter())); @@ -416,7 +417,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { public PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query, boolean ignorePermissionCheck) { return transactionTemplate.execute(status -> { EntityType entityType = resolveEntityType(query.getEntityFilter()); - QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, customerId, entityType, ignorePermissionCheck)); + SqlQueryContext ctx = new SqlQueryContext(new QueryContext(tenantId, customerId, entityType, ignorePermissionCheck)); EntityDataPageLink pageLink = query.getPageLink(); List mappings = EntityKeyMapping.prepareKeyMapping(entityType, query); @@ -524,7 +525,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { }); } - private String buildEntityWhere(QueryContext ctx, EntityFilter entityFilter, List entityFieldsFilters) { + private String buildEntityWhere(SqlQueryContext ctx, EntityFilter entityFilter, List entityFieldsFilters) { String permissionQuery = this.buildPermissionQuery(ctx, entityFilter); String entityFilterQuery = this.buildEntityFilterQuery(ctx, entityFilter); String entityFieldsQuery = EntityKeyMapping.buildQuery(ctx, entityFieldsFilters, entityFilter.getType()); @@ -538,7 +539,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return result; } - private String buildPermissionQuery(QueryContext ctx, EntityFilter entityFilter) { + private String buildPermissionQuery(SqlQueryContext ctx, EntityFilter entityFilter) { if (ctx.isIgnorePermissionCheck()) { return "1=1"; } @@ -575,7 +576,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } } - private String defaultPermissionQuery(QueryContext ctx) { + private String defaultPermissionQuery(SqlQueryContext ctx) { ctx.addUuidParameter("permissions_tenant_id", ctx.getTenantId().getId()); if (ctx.getCustomerId() != null && !ctx.getCustomerId().isNullUid()) { ctx.addUuidParameter("permissions_customer_id", ctx.getCustomerId().getId()); @@ -593,7 +594,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } } - private String buildEntityFilterQuery(QueryContext ctx, EntityFilter entityFilter) { + private String buildEntityFilterQuery(SqlQueryContext ctx, EntityFilter entityFilter) { switch (entityFilter.getType()) { case SINGLE_ENTITY: return this.singleEntityQuery(ctx, (SingleEntityFilter) entityFilter); @@ -619,7 +620,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } } - private String addEntityTableQuery(QueryContext ctx, EntityFilter entityFilter) { + private String addEntityTableQuery(SqlQueryContext ctx, EntityFilter entityFilter) { switch (entityFilter.getType()) { case RELATIONS_QUERY: return relationQuery(ctx, (RelationsQueryFilter) entityFilter); @@ -640,7 +641,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } } - private String entitySearchQuery(QueryContext ctx, EntitySearchQueryFilter entityFilter, EntityType entityType, List types) { + private String entitySearchQuery(SqlQueryContext ctx, EntitySearchQueryFilter entityFilter, EntityType entityType, List types) { EntityId rootId = entityFilter.getRootEntity(); String lvlFilter = getLvlFilter(entityFilter.getMaxLevel()); String selectFields = "SELECT tenant_id, customer_id, id, created_time, type, name, additional_info " @@ -680,7 +681,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return query; } - private String relationQuery(QueryContext ctx, RelationsQueryFilter entityFilter) { + private String relationQuery(SqlQueryContext ctx, RelationsQueryFilter entityFilter) { EntityId rootId = entityFilter.getRootEntity(); String lvlFilter = getLvlFilter(entityFilter.getMaxLevel()); String selectFields = SELECT_TENANT_ID + ", " + SELECT_CUSTOMER_ID @@ -692,6 +693,10 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { SELECT_ADDRESS + ", " + SELECT_ADDRESS_2 + ", " + SELECT_ZIP + ", " + SELECT_PHONE + ", " + SELECT_ADDITIONAL_INFO + (entityFilter.isMultiRoot() ? (", " + SELECT_RELATED_PARENT_ID) : "") + ", entity.entity_type as entity_type"; + /* + * FIXME: + * target entities are duplicated in result list, if search direction is TO and multiple relations are references to target entity + * */ String from = getQueryTemplate(entityFilter.getDirection(), entityFilter.isMultiRoot()); if (entityFilter.isMultiRoot()) { @@ -763,7 +768,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return "( " + selectFields + from + ")"; } - private String buildEtfCondition(QueryContext ctx, RelationEntityTypeFilter etf, EntitySearchDirection direction, int entityTypeFilterIdx) { + private String buildEtfCondition(SqlQueryContext ctx, RelationEntityTypeFilter etf, EntitySearchDirection direction, int entityTypeFilterIdx) { StringBuilder whereFilter = new StringBuilder(); String relationType = etf.getRelationType(); List entityTypes = etf.getEntityTypes(); @@ -812,7 +817,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return from; } - private String buildAliasWhereQuery(QueryContext ctx, EntityFilter entityFilter, List selectionMapping, String searchText) { + private String buildAliasWhereQuery(SqlQueryContext ctx, EntityFilter entityFilter, List selectionMapping, String searchText) { List aliasFiltersMapping = selectionMapping.stream().filter(mapping -> !mapping.isLatest() && mapping.getEntityKeyColumn() == null) .collect(Collectors.toList()); String entityFieldsQuery = EntityKeyMapping.buildQuery(ctx, aliasFiltersMapping, entityFilter.getType()); @@ -822,12 +827,12 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { result += " where (" + entityFieldsQuery + ")"; } if (!searchTextQuery.isEmpty()) { - result += (result.isEmpty() ? " where ": " and ") + "(" + searchTextQuery + ") "; + result += (result.isEmpty() ? " where " : " and ") + "(" + searchTextQuery + ") "; } return result; } - private String buildTextSearchQuery(QueryContext ctx, List selectionMapping, String searchText) { + private String buildTextSearchQuery(SqlQueryContext ctx, List selectionMapping, String searchText) { if (!StringUtils.isEmpty(searchText) && !selectionMapping.isEmpty()) { String sqlSearchText = "%" + searchText + "%"; ctx.addStringParameter("lowerSearchTextParam", sqlSearchText); @@ -844,17 +849,17 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } } - private String singleEntityQuery(QueryContext ctx, SingleEntityFilter filter) { + private String singleEntityQuery(SqlQueryContext ctx, SingleEntityFilter filter) { ctx.addUuidParameter("entity_filter_single_entity_id", filter.getSingleEntity().getId()); return "e.id=:entity_filter_single_entity_id"; } - private String entityListQuery(QueryContext ctx, EntityListFilter filter) { + private String entityListQuery(SqlQueryContext ctx, EntityListFilter filter) { ctx.addUuidListParameter("entity_filter_entity_ids", filter.getEntityList().stream().map(UUID::fromString).collect(Collectors.toList())); return "e.id in (:entity_filter_entity_ids)"; } - private String entityNameQuery(QueryContext ctx, EntityNameFilter filter) { + private String entityNameQuery(SqlQueryContext ctx, EntityNameFilter filter) { ctx.addStringParameter("entity_filter_name_filter", filter.getEntityNameFilter()); String nameColumn = getNameColumn(filter.getEntityType()); if (filter.getEntityNameFilter().startsWith("%") || filter.getEntityNameFilter().endsWith("%")) { @@ -864,7 +869,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return String.format("e.%s ilike concat(:entity_filter_name_filter, '%%')", nameColumn); } - private String typeQuery(QueryContext ctx, EntityFilter filter) { + private String typeQuery(SqlQueryContext ctx, EntityFilter filter) { List types; String name; String nameColumn; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java index 64e1f84db9..f6ca1581d5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java @@ -37,7 +37,7 @@ public class DefaultQueryLogComponent implements QueryLogComponent { private long logQueriesThreshold; @Override - public void logQuery(QueryContext ctx, String query, long duration) { + public void logQuery(SqlQueryContext ctx, String query, long duration) { if (logSqlQueries && duration > logQueriesThreshold) { String sqlToUse = substituteParametersInSqlString(query, ctx); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsApiService.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsApiService.java new file mode 100644 index 0000000000..e486d3b645 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsApiService.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.query; + +import com.google.common.util.concurrent.ListenableFuture; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.edqs.query.EdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsResponse; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.edqs.EdqsApiService; + +@Service +@Slf4j +@ConditionalOnMissingBean(value = EdqsApiService.class, ignored = DummyEdqsApiService.class) +public class DummyEdqsApiService implements EdqsApiService { + + @Override + public ListenableFuture processRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isEnabled() { + return false; + } + + @Override + public void setEnabled(boolean enabled) { + log.warn("Got request to enable EDQS API, but it isn't supported", new RuntimeException("stacktrace")); + } + + @Override + public boolean isSupported() { + return false; + } + + @Override + public boolean isAutoEnable() { + return false; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java new file mode 100644 index 0000000000..514e07c323 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.query; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.edqs.EdqsService; + +@Service +@ConditionalOnMissingBean(value = EdqsService.class, ignored = DummyEdqsService.class) +public class DummyEdqsService implements EdqsService { + + @Override + public void onUpdate(TenantId tenantId, EntityId entityId, Object entity) {} + + @Override + public void onUpdate(TenantId tenantId, ObjectType objectType, EdqsObject object) {} + + @Override + public void onDelete(TenantId tenantId, EntityId entityId) {} + + @Override + public void onDelete(TenantId tenantId, ObjectType objectType, EdqsObject object) {} + + @Override + public void processSystemRequest(ToCoreEdqsRequest request) {} + + @Override + public void processSystemMsg(ToCoreEdqsMsg request) {} + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java index 949941a3b6..9755201fe9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java @@ -103,7 +103,7 @@ public class EntityKeyMapping { public static final List labeledEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME, TYPE, LABEL, ADDITIONAL_INFO); public static final List contactBasedEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, EMAIL, TITLE, COUNTRY, STATE, CITY, ADDRESS, ADDRESS_2, ZIP, PHONE, ADDITIONAL_INFO); - public static final Set apiUsageStateEntityFields = new HashSet<>(Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME)); + public static final Set apiUsageStateEntityFields = new HashSet<>(Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME)); public static final Set commonEntityFieldsSet = new HashSet<>(commonEntityFields); public static final Set relationQueryEntityFieldsSet = new HashSet<>(Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME, TYPE, LABEL, FIRST_NAME, LAST_NAME, EMAIL, REGION, TITLE, COUNTRY, STATE, CITY, ADDRESS, ADDRESS_2, ZIP, PHONE, ADDITIONAL_INFO, RELATED_PARENT_ID)); @@ -265,7 +265,7 @@ public class EntityKeyMapping { return alias; } - public Stream toQueries(QueryContext ctx, EntityFilterType filterType) { + public Stream toQueries(SqlQueryContext ctx, EntityFilterType filterType) { if (hasFilter()) { String keyAlias = (entityKey.getType().equals(EntityKeyType.ENTITY_FIELD) && getEntityKeyColumn() != null) ? "e" : alias; return keyFilters.stream().map(keyFilter -> @@ -275,7 +275,7 @@ public class EntityKeyMapping { } } - public String toLatestJoin(QueryContext ctx, EntityFilter entityFilter, EntityType entityType) { + public String toLatestJoin(SqlQueryContext ctx, EntityFilter entityFilter, EntityType entityType) { String entityTypeStr; if (entityFilter.getType().equals(EntityFilterType.RELATIONS_QUERY)) { entityTypeStr = "entities.entity_type"; @@ -303,9 +303,9 @@ public class EntityKeyMapping { if (entityKey.getType().equals(EntityKeyType.CLIENT_ATTRIBUTE)) { scope = AttributeScope.CLIENT_SCOPE.getId(); } else if (entityKey.getType().equals(EntityKeyType.SHARED_ATTRIBUTE)) { - scope = AttributeScope.SHARED_SCOPE.getId();; + scope = AttributeScope.SHARED_SCOPE.getId(); ; } else { - scope = AttributeScope.SERVER_SCOPE.getId();; + scope = AttributeScope.SERVER_SCOPE.getId(); ; } query = String.format("%s AND %s.attribute_type=%s %s", query, alias, scope, filterQuery); } else { @@ -318,7 +318,7 @@ public class EntityKeyMapping { } } - private boolean hasFilterValues(QueryContext ctx) { + private boolean hasFilterValues(SqlQueryContext ctx) { return Arrays.stream(ctx.getParameterNames()).anyMatch(parameterName -> { return !parameterName.equals(getKeyId()) && parameterName.startsWith(alias); }); @@ -333,14 +333,14 @@ public class EntityKeyMapping { Collectors.joining(", ")); } - public static String buildLatestJoins(QueryContext ctx, EntityFilter entityFilter, EntityType entityType, List latestMappings, boolean countQuery) { + public static String buildLatestJoins(SqlQueryContext ctx, EntityFilter entityFilter, EntityType entityType, List latestMappings, boolean countQuery) { return latestMappings.stream() .filter(mapping -> !countQuery || mapping.hasFilter()) .map(mapping -> mapping.toLatestJoin(ctx, entityFilter, entityType)) .collect(Collectors.joining(" ")); } - public static String buildQuery(QueryContext ctx, List mappings, EntityFilterType filterType) { + public static String buildQuery(SqlQueryContext ctx, List mappings, EntityFilterType filterType) { return mappings.stream() .flatMap(mapping -> mapping.toQueries(ctx, filterType)) .filter(StringUtils::isNotEmpty) @@ -510,12 +510,12 @@ public class EntityKeyMapping { return getValueAlias() + "_so_num"; } - private String buildKeyQuery(QueryContext ctx, String alias, KeyFilter keyFilter, + private String buildKeyQuery(SqlQueryContext ctx, String alias, KeyFilter keyFilter, EntityFilterType filterType) { return this.buildPredicateQuery(ctx, alias, keyFilter.getKey(), keyFilter.getPredicate(), filterType); } - private String buildPredicateQuery(QueryContext ctx, String alias, EntityKey key, + private String buildPredicateQuery(SqlQueryContext ctx, String alias, EntityKey key, KeyFilterPredicate predicate, EntityFilterType filterType) { if (predicate.getType().equals(FilterPredicateType.COMPLEX)) { return this.buildComplexPredicateQuery(ctx, alias, key, (ComplexFilterPredicate) predicate, filterType); @@ -524,7 +524,7 @@ public class EntityKeyMapping { } } - private String buildComplexPredicateQuery(QueryContext ctx, String alias, EntityKey key, + private String buildComplexPredicateQuery(SqlQueryContext ctx, String alias, EntityKey key, ComplexFilterPredicate predicate, EntityFilterType filterType) { String result = predicate.getPredicates().stream() .map(keyFilterPredicate -> this.buildPredicateQuery(ctx, alias, key, keyFilterPredicate, filterType)) @@ -536,7 +536,7 @@ public class EntityKeyMapping { return result; } - private String buildSimplePredicateQuery(QueryContext ctx, String alias, EntityKey key, + private String buildSimplePredicateQuery(SqlQueryContext ctx, String alias, EntityKey key, KeyFilterPredicate predicate, EntityFilterType filterType) { if (key.getType().equals(EntityKeyType.ENTITY_FIELD)) { String field = (getEntityKeyColumn() != null) ? alias + "." + getEntityKeyColumn() : alias; @@ -571,7 +571,7 @@ public class EntityKeyMapping { } } - private String buildStringPredicateQuery(QueryContext ctx, String field, StringFilterPredicate stringFilterPredicate) { + private String buildStringPredicateQuery(SqlQueryContext ctx, String field, StringFilterPredicate stringFilterPredicate) { String operationField = field; String paramName = getNextParameterName(field); String value = stringFilterPredicate.getValue().getValue(); @@ -624,7 +624,7 @@ public class EntityKeyMapping { return String.format("((%s is not null and %s)", field, stringOperationQuery); } - private String buildNumericPredicateQuery(QueryContext ctx, String field, NumericFilterPredicate numericFilterPredicate) { + private String buildNumericPredicateQuery(SqlQueryContext ctx, String field, NumericFilterPredicate numericFilterPredicate) { String paramName = getNextParameterName(field); ctx.addDoubleParameter(paramName, numericFilterPredicate.getValue().getValue()); String numericOperationQuery = ""; @@ -651,7 +651,7 @@ public class EntityKeyMapping { return String.format("(%s is not null and %s)", field, numericOperationQuery); } - private String buildBooleanPredicateQuery(QueryContext ctx, String field, + private String buildBooleanPredicateQuery(SqlQueryContext ctx, String field, BooleanFilterPredicate booleanFilterPredicate) { String paramName = getNextParameterName(field); ctx.addBooleanParameter(paramName, booleanFilterPredicate.getValue().getValue()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryLogComponent.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryLogComponent.java index ea15421fb7..86daeea77d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryLogComponent.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryLogComponent.java @@ -17,5 +17,5 @@ package org.thingsboard.server.dao.sql.query; public interface QueryLogComponent { - void logQuery(QueryContext ctx, String query, long duration); + void logQuery(SqlQueryContext ctx, String query, long duration); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryContext.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/SqlQueryContext.java similarity index 94% rename from dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryContext.java rename to dao/src/main/java/org/thingsboard/server/dao/sql/query/SqlQueryContext.java index 0e33c1b44f..625458342c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryContext.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/SqlQueryContext.java @@ -21,6 +21,7 @@ import org.springframework.jdbc.core.namedparam.SqlParameterSource; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.permission.QueryContext; import java.sql.Types; import java.util.HashMap; @@ -29,14 +30,14 @@ import java.util.Map; import java.util.UUID; @Slf4j -public class QueryContext implements SqlParameterSource { +public class SqlQueryContext implements SqlParameterSource { private static final UUIDJdbcType UUID_TYPE = UUIDJdbcType.INSTANCE; - private final QuerySecurityContext securityCtx; + private final QueryContext securityCtx; private final StringBuilder query; private final Map params; - public QueryContext(QuerySecurityContext securityCtx) { + public SqlQueryContext(QueryContext securityCtx) { this.securityCtx = securityCtx; query = new StringBuilder(); params = new HashMap<>(); @@ -48,7 +49,7 @@ public class QueryContext implements SqlParameterSource { if (oldParam != null && oldParam.value != null && !oldParam.value.equals(newParam.value)) { throw new RuntimeException("Parameter with name: " + name + " was already registered!"); } - if(value == null){ + if (value == null) { log.warn("[{}][{}][{}] Trying to set null value", getTenantId(), getCustomerId(), name); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java index dcfd008367..566e4eaddc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.QueueEntity; import org.thingsboard.server.dao.queue.QueueDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -38,7 +39,7 @@ import java.util.UUID; @Slf4j @Component @SqlDao -public class JpaQueueDao extends JpaAbstractDao implements QueueDao { +public class JpaQueueDao extends JpaAbstractDao implements QueueDao, TenantEntityDao { @Autowired private QueueRepository queueRepository; @@ -87,6 +88,11 @@ public class JpaQueueDao extends JpaAbstractDao implements Q .findByTenantId(tenantId.getId(), pageLink.getTextSearch(), DaoUtil.toPageable(pageLink))); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findQueuesByTenantId(tenantId, pageLink); + } + @Override public EntityType getEntityType() { return EntityType.QUEUE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java index 15428dfa63..e09d01503c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java @@ -17,9 +17,11 @@ package org.thingsboard.server.dao.sql.queue; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.QueueStatsFields; import org.thingsboard.server.common.data.id.QueueStatsId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -60,7 +62,7 @@ public class JpaQueueStatsDao extends JpaAbstractDao findByTenantId(TenantId tenantId, PageLink pageLink) { + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { return DaoUtil.toPageData(queueStatsRepository.findByTenantId(tenantId.getId(), pageLink.getTextSearch(), DaoUtil.toPageable(pageLink))); } @@ -74,6 +76,11 @@ public class JpaQueueStatsDao extends JpaAbstractDao findNextBatch(UUID id, int batchSize) { + return queueStatsRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.QUEUE_STATS; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/QueueStatsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/QueueStatsRepository.java index bff8f05658..38e6fa9977 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/QueueStatsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/QueueStatsRepository.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.sql.queue; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; @@ -22,6 +23,7 @@ import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.edqs.fields.QueueStatsFields; import org.thingsboard.server.dao.model.sql.QueueStatsEntity; import java.util.List; @@ -45,4 +47,8 @@ public interface QueueStatsRepository extends JpaRepository findByTenantIdAndIdIn(UUID tenantId, List queueStatsIds); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.QueueStatsFields(q.id, q.createdTime," + + "q.tenantId, q.queueName, q.serviceId) FROM QueueStatsEntity q WHERE q.id > :id ORDER BY q.id") + List findNextBatch(@Param("id") UUID id, Limit limit); + } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index 97e82b2021..7417418f54 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -127,6 +127,7 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } + @Override public ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { return service.submit(() -> checkRelation(tenantId, from, to, relationType, typeGroup)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index cebd95f08e..b4e8a21372 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.sql.relation; +import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; @@ -84,4 +85,15 @@ public interface RelationRepository @Query("DELETE FROM RelationEntity r where r.fromId = :fromId and r.fromType = :fromType and r.relationTypeGroup in :relationTypeGroups") void deleteByFromIdAndFromTypeAndRelationTypeGroupIn(@Param("fromId") UUID fromId, @Param("fromType") String fromType, @Param("relationTypeGroups") List relationTypeGroups); + @Query(value = "SELECT from_id, from_type, relation_type_group, relation_type, to_id, to_type, additional_info, version FROM relation" + + " WHERE (from_id, from_type, relation_type_group, relation_type, to_id, to_type) > " + + "(:fromId, :fromType, :relationTypeGroup, :relationType, :toId, :toType) ORDER BY " + + "from_id, from_type, relation_type_group, relation_type, to_id, to_type LIMIT :batchSize", nativeQuery = true) + List findNextBatch(@Param("fromId") UUID fromId, + @Param("fromType") String fromType, + @Param("relationTypeGroup") String relationTypeGroup, + @Param("relationType") String relationType, + @Param("toId") UUID toId, + @Param("toType") String toType, + @Param("batchSize") int batchSize); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java index 15517b4e3f..6cce9d76c2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.TbResourceEntity; import org.thingsboard.server.dao.resource.TbResourceDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -38,7 +39,7 @@ import java.util.UUID; @Slf4j @Component @SqlDao -public class JpaTbResourceDao extends JpaAbstractDao implements TbResourceDao { +public class JpaTbResourceDao extends JpaAbstractDao implements TbResourceDao, TenantEntityDao { private final TbResourceRepository resourceRepository; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDao.java index ef7d28bb63..73e251b6db 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDao.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rpc.Rpc; import org.thingsboard.server.common.data.rpc.RpcStatus; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.RpcEntity; import org.thingsboard.server.dao.rpc.RpcDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -39,7 +40,7 @@ import java.util.UUID; @Component @AllArgsConstructor @SqlDao -public class JpaRpcDao extends JpaAbstractDao implements RpcDao { +public class JpaRpcDao extends JpaAbstractDao implements RpcDao, TenantEntityDao { private final RpcRepository rpcRepository; @@ -74,6 +75,11 @@ public class JpaRpcDao extends JpaAbstractDao implements RpcDao return rpcRepository.deleteOutdatedRpcByTenantId(tenantId.getId(), expirationTime); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findAllRpcByTenantId(tenantId, pageLink); + } + @Override public EntityType getEntityType() { return EntityType.RPC; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java index 3d025fc469..77044d41dc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java @@ -17,9 +17,11 @@ package org.thingsboard.server.dao.sql.rule; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.RuleChainFields; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -33,6 +35,7 @@ import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; import java.util.Collection; +import java.util.List; import java.util.Optional; import java.util.UUID; @@ -133,6 +136,16 @@ public class JpaRuleChainDao extends JpaAbstractDao return findRootRuleChainByTenantIdAndType(tenantId, RuleChainType.CORE); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findRuleChainsByTenantId(tenantId.getId(), pageLink); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return ruleChainRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.RULE_CHAIN; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java index c3a8b447c5..8ecce0672f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.RuleNodeEntity; import org.thingsboard.server.dao.rule.RuleNodeDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -40,7 +41,7 @@ import java.util.stream.Collectors; @Slf4j @Component @SqlDao -public class JpaRuleNodeDao extends JpaAbstractDao implements RuleNodeDao { +public class JpaRuleNodeDao extends JpaAbstractDao implements RuleNodeDao, TenantEntityDao { @Autowired private RuleNodeRepository ruleNodeRepository; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java index 01bec2a846..cfa06caf14 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.dao.sql.rule; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.RuleChainFields; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.RuleChainEntity; @@ -70,4 +72,7 @@ public interface RuleChainRepository extends JpaRepository :id ORDER BY r.id") + List findNextBatch(@Param("id") UUID id, Limit limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java index bd9b20e01e..c27602421d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.settings; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.dao.model.sql.AdminSettingsEntity; @@ -33,4 +35,6 @@ public interface AdminSettingsRepository extends JpaRepository findByTenantId(UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java index 94f39d223c..68ce5e9d22 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java @@ -21,7 +21,11 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.AdminSettingsEntity; import org.thingsboard.server.dao.settings.AdminSettingsDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -32,7 +36,7 @@ import java.util.UUID; @Component @SqlDao @Slf4j -public class JpaAdminSettingsDao extends JpaAbstractDao implements AdminSettingsDao { +public class JpaAdminSettingsDao extends JpaAbstractDao implements AdminSettingsDao, TenantEntityDao { @Autowired private AdminSettingsRepository adminSettingsRepository; @@ -68,4 +72,9 @@ public class JpaAdminSettingsDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(adminSettingsRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java index 0ba805f6e1..d031dbb8ee 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java @@ -16,11 +16,13 @@ package org.thingsboard.server.dao.sql.tenant; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantInfo; +import org.thingsboard.server.common.data.edqs.fields.TenantFields; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.page.PageData; @@ -94,4 +96,9 @@ public class JpaTenantDao extends JpaAbstractDao implement .map(TenantId::fromUUID) .collect(Collectors.toList()); } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return tenantRepository.findNextBatch(id, Limit.of(batchSize)); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantProfileDao.java index acc5feef31..839d62c48d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantProfileDao.java @@ -16,11 +16,13 @@ package org.thingsboard.server.dao.sql.tenant; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.edqs.fields.TenantProfileFields; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -87,6 +89,11 @@ public class JpaTenantProfileDao extends JpaAbstractDao findNextBatch(UUID id, int batchSize) { + return tenantProfileRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.TENANT_PROFILE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantProfileRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantProfileRepository.java index dc918c900e..c5759c8a0f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantProfileRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantProfileRepository.java @@ -15,12 +15,14 @@ */ package org.thingsboard.server.dao.sql.tenant; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.EntityInfo; +import org.thingsboard.server.common.data.edqs.fields.TenantProfileFields; import org.thingsboard.server.dao.model.sql.TenantProfileEntity; import java.util.List; @@ -55,4 +57,8 @@ public interface TenantProfileRepository extends JpaRepository findByIdIn(List ids); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.TenantProfileFields(t.id, t.createdTime, t.name," + + "t.isDefault) FROM TenantProfileEntity t WHERE t.id > :id ORDER BY t.id") + List findNextBatch(@Param("id") UUID id, Limit limit); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java index 8adb4e0261..bafa9a6fe6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.dao.sql.tenant; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.TenantFields; import org.thingsboard.server.dao.model.sql.TenantEntity; import org.thingsboard.server.dao.model.sql.TenantInfoEntity; @@ -53,4 +55,8 @@ public interface TenantRepository extends JpaRepository { @Query("SELECT t.id FROM TenantEntity t where t.tenantProfileId = :tenantProfileId") List findTenantIdsByTenantProfileId(@Param("tenantProfileId") UUID tenantProfileId); + + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.TenantFields(t.id, t.createdTime, t.title, t.version," + + "t.additionalInfo, t.country, t.state, t.city, t.address, t.address2, t.zip, t.phone, t.email, t.region) FROM TenantEntity t WHERE t.id > :id ORDER BY t.id") + List findNextBatch(@Param("id") UUID id, Limit limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java index 0b27632f7a..98e62fc110 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java @@ -15,13 +15,18 @@ */ package org.thingsboard.server.dao.sql.usagerecord; +import org.springframework.data.domain.Limit; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.edqs.fields.ApiUsageStateFields; import org.thingsboard.server.dao.model.sql.ApiUsageStateEntity; +import java.util.List; import java.util.UUID; /** @@ -35,6 +40,8 @@ public interface ApiUsageStateRepository extends JpaRepository findAllByTenantId(UUID tenantId, Pageable pageable); + @Transactional @Modifying @Query("DELETE FROM ApiUsageStateEntity ur WHERE ur.tenantId = :tenantId") @@ -44,4 +51,10 @@ public interface ApiUsageStateRepository extends JpaRepository :id ORDER BY a.id") + List findNextBatch(@Param("id") UUID id, Limit limit); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java index 6c0ac91507..ec68fa34c3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java @@ -15,18 +15,23 @@ */ package org.thingsboard.server.dao.sql.usagerecord; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.ApiUsageStateFields; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.ApiUsageStateEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.usagerecord.ApiUsageStateDao; import org.thingsboard.server.dao.util.SqlDao; +import java.util.List; import java.util.UUID; /** @@ -72,6 +77,16 @@ public class JpaApiUsageStateDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(apiUsageStateRepository.findAllByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return apiUsageStateRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.API_USAGE_STATE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java index d68dd395bd..4f9ff5d222 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.UserAuthSettings; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.UserAuthSettingsEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.user.UserAuthSettingsDao; @@ -31,7 +32,7 @@ import java.util.UUID; @Component @RequiredArgsConstructor @SqlDao -public class JpaUserAuthSettingsDao extends JpaAbstractDao implements UserAuthSettingsDao { +public class JpaUserAuthSettingsDao extends JpaAbstractDao implements UserAuthSettingsDao, TenantEntityDao { private final UserAuthSettingsRepository repository; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserCredentialsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserCredentialsDao.java index 30b643afa5..bcae1dcdf4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserCredentialsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserCredentialsDao.java @@ -20,8 +20,11 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.UserCredentialsEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.user.UserCredentialsDao; @@ -34,7 +37,7 @@ import java.util.UUID; */ @Component @SqlDao -public class JpaUserCredentialsDao extends JpaAbstractDao implements UserCredentialsDao { +public class JpaUserCredentialsDao extends JpaAbstractDao implements UserCredentialsDao, TenantEntityDao { @Autowired private UserCredentialsRepository userCredentialsRepository; @@ -84,4 +87,9 @@ public class JpaUserCredentialsDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(userCredentialsRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java index c0e5b530ef..35d15bab51 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java @@ -16,10 +16,12 @@ package org.thingsboard.server.dao.sql.user; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.edqs.fields.UserFields; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; @@ -139,6 +141,16 @@ public class JpaUserDao extends JpaAbstractDao implements User return userRepository.countByTenantId(tenantId.getId()); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return userRepository.findNextBatch(id, Limit.of(batchSize)); + } + @Override public EntityType getEntityType() { return EntityType.USER; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java index bfe0e60556..646b11975e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java @@ -20,12 +20,14 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.settings.UserSettings; import org.thingsboard.server.common.data.settings.UserSettingsCompositeKey; import org.thingsboard.server.common.data.settings.UserSettingsType; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.UserSettingsEntity; -import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import org.thingsboard.server.dao.user.UserSettingsDao; import org.thingsboard.server.dao.util.SqlDao; @@ -34,7 +36,7 @@ import java.util.List; @Slf4j @Component @SqlDao -public class JpaUserSettingsDao extends JpaAbstractDaoListeningExecutorService implements UserSettingsDao { +public class JpaUserSettingsDao implements UserSettingsDao, TenantEntityDao { @Autowired private UserSettingsRepository userSettingsRepository; @@ -66,4 +68,9 @@ public class JpaUserSettingsDao extends JpaAbstractDaoListeningExecutorService i return DaoUtil.convertDataList(userSettingsRepository.findByTypeAndPathExisting(type.name(), path)); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(userSettingsRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java index 7ecb45d5e1..849cb22496 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.user; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; @@ -35,4 +37,7 @@ public interface UserAuthSettingsRepository extends JpaRepository findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java index 0dd3462c8b..51bc8702bc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java @@ -15,9 +15,12 @@ */ package org.thingsboard.server.dao.sql.user; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.UserCredentialsEntity; @@ -52,4 +55,7 @@ public interface UserCredentialsRepository extends JpaRepository findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java index 731eda205e..0a30a859c6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java @@ -15,15 +15,18 @@ */ package org.thingsboard.server.dao.sql.user; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.UserFields; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.model.sql.UserEntity; import java.util.Collection; +import java.util.List; import java.util.UUID; /** @@ -71,4 +74,8 @@ public interface UserRepository extends JpaRepository { Long countByTenantId(UUID tenantId); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.UserFields(u.id, u.createdTime, u.tenantId," + + "u.customerId, u.version, u.firstName, u.lastName, u.email, u.phone, u.additionalInfo) " + + "FROM UserEntity u WHERE u.id > :id ORDER BY u.id") + List findNextBatch(@Param("id") UUID id, Limit limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java index 12d43ff5ac..9423baafc0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.user; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; @@ -36,4 +38,7 @@ public interface UserSettingsRepository extends JpaRepository findByTypeAndPathExisting(@Param("type") String type, @Param("path") String[] path); + @Query("SELECT s FROM UserSettingsEntity s WHERE s.userId IN (SELECT u.id FROM UserEntity u WHERE u.tenantId = :tenantId)") + Page findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java index 5d207800d7..c728f5d006 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java @@ -16,9 +16,11 @@ package org.thingsboard.server.dao.sql.widget; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.WidgetTypeFields; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.WidgetTypeId; import org.thingsboard.server.common.data.page.PageData; @@ -30,6 +32,7 @@ import org.thingsboard.server.common.data.widget.WidgetTypeFilter; import org.thingsboard.server.common.data.widget.WidgetTypeInfo; import org.thingsboard.server.common.data.widget.WidgetsBundleWidget; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.WidgetTypeDetailsEntity; import org.thingsboard.server.dao.model.sql.WidgetTypeInfoEntity; import org.thingsboard.server.dao.model.sql.WidgetsBundleWidgetCompositeKey; @@ -53,7 +56,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; */ @Component @SqlDao -public class JpaWidgetTypeDao extends JpaAbstractDao implements WidgetTypeDao { +public class JpaWidgetTypeDao extends JpaAbstractDao implements WidgetTypeDao, TenantEntityDao { @Autowired private WidgetTypeRepository widgetTypeRepository; @@ -190,11 +193,6 @@ public class JpaWidgetTypeDao extends JpaAbstractDao findWidgetTypesNamesByTenantIdAndResourceLink(UUID tenantId, String link) { - return widgetTypeRepository.findNamesByTenantIdAndResourceLink(tenantId, link); - } - @Override public List findWidgetTypeIdsByTenantIdAndFqns(UUID tenantId, List widgetFqns) { var idFqnPairs = widgetTypeRepository.findWidgetTypeIdsByTenantIdAndFqns(tenantId, widgetFqns); @@ -260,10 +258,29 @@ public class JpaWidgetTypeDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return widgetTypeRepository.findNextBatch(id, Limit.of(batchSize)); + } + + @Override + public List findByTenantIdAndResourceLink(TenantId tenantId, String link, int limit) { + return DaoUtil.convertDataList(widgetTypeInfoRepository.findWidgetTypeInfosByTenantIdAndResourceLink(tenantId.getId(), link, limit)); + } + + @Override + public List findByResourceLink(String link, int limit) { + return DaoUtil.convertDataList(widgetTypeInfoRepository.findWidgetTypeInfosByResourceLink(link, limit)); + } + @Override public EntityType getEntityType() { return EntityType.WIDGET_TYPE; } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java index 9b78ef60ef..bc81be054a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java @@ -16,9 +16,11 @@ package org.thingsboard.server.dao.sql.widget; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Limit; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.WidgetsBundleId; import org.thingsboard.server.common.data.page.PageData; @@ -26,6 +28,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.common.data.widget.WidgetsBundleFilter; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.WidgetsBundleEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; @@ -44,7 +47,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; */ @Component @SqlDao -public class JpaWidgetsBundleDao extends JpaAbstractDao implements WidgetsBundleDao { +public class JpaWidgetsBundleDao extends JpaAbstractDao implements WidgetsBundleDao, TenantEntityDao { @Autowired private WidgetsBundleRepository widgetsBundleRepository; @@ -155,7 +158,17 @@ public class JpaWidgetsBundleDao extends JpaAbstractDao findByImageLink(String imageUrl, int limit) { - return DaoUtil.convertDataList(widgetsBundleRepository.findByImageUrl(imageUrl, limit)); + return DaoUtil.convertDataList(widgetsBundleRepository.findByImageUrl(imageUrl, limit)); + } + + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public List findNextBatch(UUID id, int batchSize) { + return widgetsBundleRepository.findNextBatch(id, Limit.of(batchSize)); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeInfoRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeInfoRepository.java index 39ba949ca9..dc79280bcf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeInfoRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeInfoRepository.java @@ -204,13 +204,20 @@ public interface WidgetTypeInfoRepository extends JpaRepository findByTenantAndImageUrl(@Param("tenantId") UUID tenantId, @Param("imageLink") String imageLink, @Param("lmt") int lmt); + List findByTenantAndImageUrl(@Param("tenantId") UUID tenantId, @Param("imageLink") String imageLink, @Param("limit") int limit); @Query(nativeQuery = true, value = "SELECT * FROM widget_type_info_view wti WHERE wti.id IN " + - "(select id from widget_type where image = :imageLink or descriptor ILIKE CONCAT('%', :imageLink, '%') limit :lmt)" + "(select id from widget_type where image = :imageLink or descriptor ILIKE CONCAT('%', :imageLink, '%') limit :limit)" ) - List findByImageUrl(@Param("imageLink") String imageLink, @Param("lmt") int lmt); + List findByImageUrl(@Param("imageLink") String imageLink, @Param("limit") int limit); + + @Query(value = "SELECT * FROM widget_type_info_view w WHERE w.tenant_id = :tenantId AND w.descriptor ILIKE CONCAT('%', :link, '%') LIMIT :limit ", nativeQuery = true) + List findWidgetTypeInfosByTenantIdAndResourceLink(@Param("tenantId") UUID tenantId, @Param("link") String link, @Param("limit") int limit); + + @Query(value = "SELECT * FROM widget_type_info_view w WHERE w.descriptor ILIKE CONCAT('%', :link, '%') LIMIT :limit ", nativeQuery = true) + List findWidgetTypeInfosByResourceLink(@Param("link") String link, @Param("limit") int limit); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java index d92bd085cf..ffa88b4d54 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.dao.sql.widget; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.WidgetTypeFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.WidgetTypeDetailsEntity; import org.thingsboard.server.dao.model.sql.WidgetTypeEntity; @@ -69,12 +71,6 @@ public interface WidgetTypeRepository extends JpaRepository> 'resources' LIKE concat('%', :resourceLink, '%')", - nativeQuery = true) - List findNamesByTenantIdAndResourceLink(@Param("tenantId") UUID tenantId, - @Param("resourceLink") String resourceLink); - @Query("SELECT externalId FROM WidgetTypeDetailsEntity WHERE id = :id") UUID getExternalIdById(@Param("id") UUID id); @@ -84,4 +80,7 @@ public interface WidgetTypeRepository extends JpaRepository findIdsByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.WidgetTypeFields(w.id, w.createdTime, w.tenantId," + + "w.name, w.version) FROM WidgetTypeEntity w WHERE w.id > :id ORDER BY w.id") + List findNextBatch(@Param("id") UUID id, Limit limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java index 516d6a6642..de778588dd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java @@ -15,11 +15,14 @@ */ package org.thingsboard.server.dao.sql.widget; +import org.springframework.data.domain.Limit; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.WidgetTypeFields; +import org.thingsboard.server.common.data.edqs.fields.WidgetsBundleFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.WidgetsBundleEntity; @@ -139,4 +142,8 @@ public interface WidgetsBundleRepository extends JpaRepository findByImageUrl(@Param("imageLink") String imageLink, @Param("lmt") int lmt); + + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.WidgetsBundleFields(w.id, w.createdTime, w.tenantId," + + "w.alias, w.version) FROM WidgetsBundleEntity w WHERE w.id > :id ORDER BY w.id") + List findNextBatch(@Param("id") UUID id, Limit limit); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java index 664da621a5..6913aa8ea6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java @@ -39,7 +39,6 @@ import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -@SuppressWarnings("UnstableApiUsage") @Slf4j public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseriesDao implements AggregationTimeseriesDao { @@ -119,4 +118,5 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries protected int getDataPointDays(TsKvEntry tsKvEntry, long ttl) { return tsKvEntry.getDataPoints() * Math.max(1, (int) (ttl / SECONDS_IN_DAY)); } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java index be5d74f758..6e43034a44 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java @@ -33,14 +33,18 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.stats.DefaultCounter; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.cache.CacheExecutorService; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; import org.thingsboard.server.dao.timeseries.TsLatestCacheKey; import org.thingsboard.server.dao.util.SqlTsLatestAnyDaoCachedRedis; import java.util.List; +import java.util.Map; import java.util.Optional; @Slf4j @@ -167,4 +171,5 @@ public class CachedRedisSqlTimeseriesLatestDao extends BaseAbstractSqlTimeseries return sqlDao.findAllKeysByEntityIds(tenantId, entityIds); } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index 5bb1c15e33..e8ef37b3b5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -24,6 +24,7 @@ import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.domain.Page; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -37,6 +38,8 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; @@ -185,6 +188,7 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme return tsKvLatestRepository.findAllKeysByEntityIds(entityIds.stream().map(EntityId::getId).collect(Collectors.toList())); } + private ListenableFuture getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { ListenableFuture> future = findNewLatestEntryFuture(tenantId, entityId, query); return Futures.transformAsync(future, entryList -> { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java index 93ad8e8beb..53a824f026 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java @@ -22,6 +22,9 @@ import org.springframework.dao.DataIntegrityViolationException; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; @@ -92,4 +95,9 @@ public class JpaKeyDictionaryDao extends JpaAbstractDaoListeningExecutorService return byKeyId.map(KeyDictionaryEntry::getKey).orElse(null); } + @Override + public PageData findAll(PageLink pageLink) { + return DaoUtil.pageToPageData(keyDictionaryRepository.findAll(DaoUtil.toPageable(pageLink))); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/KeyDictionaryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/KeyDictionaryRepository.java index 13d7481b00..d264cd9966 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/KeyDictionaryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/KeyDictionaryRepository.java @@ -15,7 +15,10 @@ */ package org.thingsboard.server.dao.sqlts.dictionary; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; @@ -25,5 +28,7 @@ public interface KeyDictionaryRepository extends JpaRepository findByKeyId(int keyId); + @Query("SELECT e FROM KeyDictionaryEntry e ORDER BY e.keyId ASC") + Page findAll(Pageable pageable); } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java index 77db6cd734..29d4377485 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java @@ -41,4 +41,10 @@ public interface TsKvLatestRepository extends JpaRepository findAllKeysByEntityIds(@Param("entityIds") List entityIds); + @Query(value = "SELECT entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v, version FROM ts_kv_latest WHERE (entity_id, key) > " + + "(:entityId, :key) ORDER BY entity_id, key LIMIT :batchSize", nativeQuery = true) + List findNextBatch(@Param("entityId") UUID entityId, + @Param("key") int key, + @Param("batchSize") int batchSize); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantDao.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantDao.java index e736bc791c..2a8269b15d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantDao.java @@ -37,10 +37,10 @@ public interface TenantDao extends Dao { * @return saved tenant object */ Tenant save(TenantId tenantId, Tenant tenant); - + /** * Find tenants by page link. - * + * * @param pageLink the page link * @return the list of tenant objects */ @@ -51,4 +51,5 @@ public interface TenantDao extends Dao { PageData findTenantsIds(PageLink pageLink); List findTenantIdsByTenantProfileId(TenantProfileId tenantProfileId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index a9dc784132..19ce8e8993 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -26,6 +26,8 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityViewId; @@ -36,8 +38,10 @@ import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.service.Validator; @@ -54,7 +58,6 @@ import static org.thingsboard.server.common.data.StringUtils.isBlank; /** * @author Andrew Shvayka */ -@SuppressWarnings("UnstableApiUsage") @Service @Slf4j public class BaseTimeseriesService implements TimeseriesService { @@ -89,6 +92,9 @@ public class BaseTimeseriesService implements TimeseriesService { @Autowired private EntityViewService entityViewService; + @Autowired + private EdqsService edqsService; + @Override public ListenableFuture> findAllByQueries(TenantId tenantId, EntityId entityId, List queries) { validate(entityId); @@ -156,60 +162,51 @@ public class BaseTimeseriesService implements TimeseriesService { } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { validate(entityId); - List> futures = new ArrayList<>(INSERTS_PER_ENTRY); - saveAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, 0L); - return Futures.transform(Futures.allAsList(futures), SUM_ALL_INTEGERS, MoreExecutors.directExecutor()); + return doSave(tenantId, entityId, List.of(tsKvEntry), 0L, true, true); } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { - return doSave(tenantId, entityId, tsKvEntries, ttl, true); + public ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { + return doSave(tenantId, entityId, tsKvEntries, ttl, true, true); } @Override - public ListenableFuture saveWithoutLatest(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { - return doSave(tenantId, entityId, tsKvEntries, ttl, false); - } - - private ListenableFuture doSave(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl, boolean saveLatest) { - int inserts = saveLatest ? INSERTS_PER_ENTRY : INSERTS_PER_ENTRY_WITHOUT_LATEST; - List> futures = new ArrayList<>(tsKvEntries.size() * inserts); - for (TsKvEntry tsKvEntry : tsKvEntries) { - if (saveLatest) { - saveAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, ttl); - } else { - saveWithoutLatestAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, ttl); - } - } - return Futures.transform(Futures.allAsList(futures), SUM_ALL_INTEGERS, MoreExecutors.directExecutor()); + public ListenableFuture saveWithoutLatest(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { + return doSave(tenantId, entityId, tsKvEntries, ttl, false, true); } @Override - public ListenableFuture> saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntries) { - List> futures = new ArrayList<>(tsKvEntries.size()); - for (TsKvEntry tsKvEntry : tsKvEntries) { - futures.add(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry)); - } - return Futures.allAsList(futures); + public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntries) { + return doSave(tenantId, entityId, tsKvEntries, 0L, true, false); } - private void saveAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { - doSaveAndRegisterFuturesFor(tenantId, futures, entityId, tsKvEntry, ttl); - futures.add(Futures.transform(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry), v -> 0, MoreExecutors.directExecutor())); - } - - private void saveWithoutLatestAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { - doSaveAndRegisterFuturesFor(tenantId, futures, entityId, tsKvEntry, ttl); - } - - private void doSaveAndRegisterFuturesFor(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { - if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW)) { + private ListenableFuture doSave(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl, boolean saveLatest, boolean saveTs) { + if (saveTs && entityId.getEntityType().equals(EntityType.ENTITY_VIEW)) { throw new IncorrectParameterException("Telemetry data can't be stored for entity view. Read only"); } - futures.add(timeseriesDao.savePartition(tenantId, entityId, tsKvEntry.getTs(), tsKvEntry.getKey())); - futures.add(timeseriesDao.save(tenantId, entityId, tsKvEntry, ttl)); + List> tsFutures = saveTs ? new ArrayList<>(tsKvEntries.size() * INSERTS_PER_ENTRY_WITHOUT_LATEST) : null; + List> latestFutures = saveLatest ? new ArrayList<>(tsKvEntries.size()) : null; + for (TsKvEntry tsKvEntry : tsKvEntries) { + if (saveTs) { + tsFutures.add(timeseriesDao.savePartition(tenantId, entityId, tsKvEntry.getTs(), tsKvEntry.getKey())); + tsFutures.add(timeseriesDao.save(tenantId, entityId, tsKvEntry, ttl)); + } + if (saveLatest) { + latestFutures.add(Futures.transform(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry), version -> { + edqsService.onUpdate(tenantId, ObjectType.LATEST_TS_KV, new LatestTsKv(entityId, tsKvEntry, version)); + return version; + }, MoreExecutors.directExecutor())); + } + } + ListenableFuture dpsFuture = saveTs ? Futures.transform(Futures.allAsList(tsFutures), SUM_ALL_INTEGERS, MoreExecutors.directExecutor()) : Futures.immediateFuture(0); + ListenableFuture> versionsFuture = saveLatest ? Futures.allAsList(latestFutures) : Futures.immediateFuture(null); + return Futures.whenAllComplete(dpsFuture, versionsFuture).call(() -> { + Integer dataPoints = Futures.getUnchecked(dpsFuture); + List versions = Futures.getUnchecked(versionsFuture); + return TimeseriesSaveResult.of(dataPoints, versions); + }, MoreExecutors.directExecutor()); } private List updateQueriesForEntityView(EntityView entityView, List queries) { @@ -248,7 +245,7 @@ public class BaseTimeseriesService implements TimeseriesService { List> futures = new ArrayList<>(keys.size()); for (String key : keys) { DeleteTsKvQuery query = new BaseDeleteTsKvQuery(key, 0, System.currentTimeMillis(), false); - futures.add(timeseriesLatestDao.removeLatest(tenantId, entityId, query)); + futures.add(doRemove(tenantId, entityId, query)); } return Futures.allAsList(futures); } @@ -269,10 +266,20 @@ public class BaseTimeseriesService implements TimeseriesService { private void deleteAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, DeleteTsKvQuery query) { futures.add(Futures.transform(timeseriesDao.remove(tenantId, entityId, query), v -> null, MoreExecutors.directExecutor())); if (query.getDeleteLatest()) { - futures.add(timeseriesLatestDao.removeLatest(tenantId, entityId, query)); + futures.add(doRemove(tenantId, entityId, query)); } } + private ListenableFuture doRemove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + return Futures.transform(timeseriesLatestDao.removeLatest(tenantId, entityId, query), result -> { + if (result.isRemoved()) { + Long version = result.getVersion(); + edqsService.onDelete(tenantId, ObjectType.LATEST_TS_KV, new LatestTsKv(entityId, query.getKey(), version)); + } + return result; + }, MoreExecutors.directExecutor()); + } + private static void validate(EntityId entityId) { Validator.validateEntityId(entityId, id -> "Incorrect entityId " + id); } @@ -302,4 +309,5 @@ public class BaseTimeseriesService implements TimeseriesService { throw new IncorrectParameterException("Incorrect DeleteTsKvQuery. Key can't be empty"); } } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java index d4b31f4b92..54a7e68725 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java @@ -36,13 +36,17 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.util.NoSqlTsLatestDao; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; @@ -99,6 +103,7 @@ public class CassandraBaseTimeseriesLatestDao extends AbstractCassandraBaseTimes return Collections.emptyList(); } + @Override public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(getLatestStmt().bind()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java index 7f7fe88936..32479301ae 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java @@ -22,8 +22,12 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import java.util.List; +import java.util.Map; import java.util.Optional; public interface TimeseriesLatestDao { @@ -49,4 +53,5 @@ public interface TimeseriesLatestDao { List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId); List findAllKeysByEntityIds(TenantId tenantId, List entityIds); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateDao.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateDao.java index 29fe557822..ffff210693 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateDao.java @@ -19,10 +19,11 @@ import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.TenantEntityDao; import java.util.UUID; -public interface ApiUsageStateDao extends Dao { +public interface ApiUsageStateDao extends Dao, TenantEntityDao { /** * Save or update usage record object @@ -50,4 +51,5 @@ public interface ApiUsageStateDao extends Dao { void deleteApiUsageStateByTenantId(TenantId tenantId); void deleteApiUsageStateByEntityId(EntityId entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java index 3a867587e1..1282493650 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -113,6 +113,14 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A ApiUsageState saved = apiUsageStateDao.save(apiUsageState.getTenantId(), apiUsageState); + eventPublisher.publishEvent(SaveEntityEvent.builder() + .tenantId(saved.getTenantId()) + .entityId(saved.getId()) + .entity(saved) + .created(true) + .broadcastEvent(false) + .build()); + List apiUsageStates = new ArrayList<>(); apiUsageStates.add(new BasicTsKvEntry(saved.getCreatedTime(), new StringDataEntry(ApiFeature.TRANSPORT.getApiStateKey(), ApiUsageStateValue.ENABLED.name()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java index c1d2a3f2e1..b60b263ac8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java @@ -28,7 +28,7 @@ import org.thingsboard.server.dao.TenantEntityDao; import java.util.List; import java.util.UUID; -public interface UserDao extends Dao, TenantEntityDao { +public interface UserDao extends Dao, TenantEntityDao { /** * Save or update user object diff --git a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeDao.java b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeDao.java index 7302ed41bc..bbb8d026c0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeDao.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.widget.WidgetsBundleWidget; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.ImageContainerDao; +import org.thingsboard.server.dao.ResourceContainerDao; import java.util.List; import java.util.UUID; @@ -35,7 +36,7 @@ import java.util.UUID; /** * The Interface WidgetTypeDao. */ -public interface WidgetTypeDao extends Dao, ExportableEntityDao, ImageContainerDao { +public interface WidgetTypeDao extends Dao, ExportableEntityDao, ImageContainerDao, ResourceContainerDao { /** * Save or update widget type object @@ -97,8 +98,6 @@ public interface WidgetTypeDao extends Dao, ExportableEntityD WidgetTypeDetails findDetailsByTenantIdAndFqn(UUID tenantId, String fqn); - List findWidgetTypesNamesByTenantIdAndResourceLink(UUID tenantId, String link); - List findWidgetTypeIdsByTenantIdAndFqns(UUID tenantId, List widgetFqns); List findWidgetsBundleWidgetsByWidgetsBundleId(UUID tenantId, UUID widgetsBundleId); diff --git a/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql b/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql index b0c69ef6f3..b756d282f8 100644 --- a/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql +++ b/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql @@ -36,3 +36,5 @@ CREATE INDEX IF NOT EXISTS idx_lc_event_main CREATE INDEX IF NOT EXISTS idx_error_event_main ON error_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); +CREATE INDEX IF NOT EXISTS idx_cf_debug_event_main + ON cf_debug_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index b88c6bba50..b425550e7e 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -907,3 +907,44 @@ CREATE TABLE IF NOT EXISTS qr_code_settings ( qr_code_config VARCHAR(100000), CONSTRAINT qr_code_settings_tenant_id_unq_key UNIQUE (tenant_id) ); + +CREATE TABLE IF NOT EXISTS calculated_field ( + id uuid NOT NULL CONSTRAINT calculated_field_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid NOT NULL, + entity_type VARCHAR(32), + entity_id uuid NOT NULL, + type varchar(32) NOT NULL, + name varchar(255) NOT NULL, + configuration_version int DEFAULT 0, + configuration varchar(1000000), + version BIGINT DEFAULT 1, + debug_settings varchar(1024), + CONSTRAINT calculated_field_unq_key UNIQUE (entity_id, name) +); + +CREATE TABLE IF NOT EXISTS calculated_field_link ( + id uuid NOT NULL CONSTRAINT calculated_field_link_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid NOT NULL, + entity_type VARCHAR(32), + entity_id uuid NOT NULL, + calculated_field_id uuid NOT NULL, + CONSTRAINT fk_calculated_field_id FOREIGN KEY (calculated_field_id) REFERENCES calculated_field(id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS cf_debug_event ( + id uuid NOT NULL, + tenant_id uuid NOT NULL , + ts bigint NOT NULL, + entity_id uuid NOT NULL, -- calculated field id + service_id varchar, + cf_id uuid NOT NULL, + e_entity_id uuid, -- target entity id + e_entity_type varchar, + e_msg_id uuid, + e_msg_type varchar, + e_args varchar, + e_result varchar, + e_error varchar +) PARTITION BY RANGE (ts); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java index fbad72f371..25339244fd 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java @@ -77,7 +77,6 @@ import java.util.UUID; import static org.junit.Assert.assertNotNull; - @RunWith(SpringRunner.class) @ContextConfiguration(classes = AbstractServiceTest.class, loader = AnnotationConfigContextLoader.class) @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) @@ -131,7 +130,7 @@ public abstract class AbstractServiceTest { } public JsonNode readFromResource(String resourceName) throws IOException { - try (InputStream is = this.getClass().getClassLoader().getResourceAsStream(resourceName)){ + try (InputStream is = this.getClass().getClassLoader().getResourceAsStream(resourceName)) { return JacksonUtil.fromBytes(Objects.requireNonNull(is).readAllBytes()); } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AlarmServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AlarmServiceTest.java index c82c6bd5f6..849528b091 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AlarmServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AlarmServiceTest.java @@ -22,6 +22,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmApiCallResult; @@ -48,6 +49,7 @@ import org.thingsboard.server.common.data.query.DeviceTypeFilter; import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityListFilter; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.security.Authority; @@ -936,4 +938,53 @@ public class AlarmServiceTest extends AbstractServiceTest { Assert.assertEquals(0, alarms.getData().size()); } + @Test + public void testCountAlarmsForEntities() throws ExecutionException, InterruptedException { + AssetId parentId = new AssetId(Uuids.timeBased()); + AssetId childId = new AssetId(Uuids.timeBased()); + + EntityRelation relation = new EntityRelation(parentId, childId, EntityRelation.CONTAINS_TYPE); + + Assert.assertTrue(relationService.saveRelationAsync(tenantId, relation).get()); + + long ts = System.currentTimeMillis(); + AlarmApiCallResult result = alarmService.createAlarm(AlarmCreateOrUpdateActiveRequest.builder() + .tenantId(tenantId) + .originator(childId) + .type(TEST_ALARM) + .severity(AlarmSeverity.CRITICAL) + .startTs(ts).build()); + AlarmInfo created = result.getAlarm(); + created.setPropagate(true); + result = alarmService.updateAlarm(AlarmUpdateRequest.fromAlarm(created)); + created = result.getAlarm(); + + EntityListFilter entityListFilter = new EntityListFilter(); + entityListFilter.setEntityList(List.of(childId.getId().toString(), parentId.getId().toString())); + entityListFilter.setEntityType(EntityType.ASSET); + AlarmCountQuery countQuery = new AlarmCountQuery(entityListFilter); + countQuery.setStartTs(0L); + countQuery.setEndTs(System.currentTimeMillis()); + + long alarmsCount = alarmService.countAlarmsByQuery(tenantId, null, countQuery, List.of(childId)); + Assert.assertEquals(1, alarmsCount); + + countQuery.setSearchPropagatedAlarms(true); + + alarmsCount = alarmService.countAlarmsByQuery(tenantId, null, countQuery, List.of(parentId)); + Assert.assertEquals(1, alarmsCount); + + created = alarmService.acknowledgeAlarm(tenantId, created.getId(), System.currentTimeMillis()).getAlarm(); + + countQuery.setStatusList(List.of(AlarmSearchStatus.UNACK)); + alarmsCount = alarmService.countAlarmsByQuery(tenantId, null, countQuery, List.of(childId)); + Assert.assertEquals(0, alarmsCount); + + alarmService.clearAlarm(tenantId, created.getId(), System.currentTimeMillis(), null); + + countQuery.setStatusList(List.of(AlarmSearchStatus.CLEARED)); + alarmsCount = alarmService.countAlarmsByQuery(tenantId, null, countQuery, List.of(childId)); + Assert.assertEquals(1, alarmsCount); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index bcbc0f9be1..462e7a894c 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -30,6 +30,14 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -39,6 +47,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.relation.RelationService; @@ -46,7 +55,9 @@ import org.thingsboard.server.dao.relation.RelationService; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @DaoSqlTest @@ -63,6 +74,8 @@ public class AssetServiceTest extends AbstractServiceTest { @Autowired private AssetProfileService assetProfileService; @Autowired + private CalculatedFieldService calculatedFieldService; + @Autowired private PlatformTransactionManager platformTransactionManager; private IdComparator idComparator = new IdComparator<>(); @@ -214,24 +227,24 @@ public class AssetServiceTest extends AbstractServiceTest { public void testFindAssetTypesByTenantId() throws Exception { List assets = new ArrayList<>(); try { - for (int i=0;i<3;i++) { + for (int i = 0; i < 3; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("My asset B"+i); + asset.setName("My asset B" + i); asset.setType("typeB"); assets.add(assetService.saveAsset(asset)); } - for (int i=0;i<7;i++) { + for (int i = 0; i < 7; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("My asset C"+i); + asset.setName("My asset C" + i); asset.setType("typeC"); assets.add(assetService.saveAsset(asset)); } - for (int i=0;i<9;i++) { + for (int i = 0; i < 9; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("My asset A"+i); + asset.setName("My asset A" + i); asset.setType("typeA"); assets.add(assetService.saveAsset(asset)); } @@ -242,7 +255,9 @@ public class AssetServiceTest extends AbstractServiceTest { Assert.assertEquals("typeB", assetTypes.get(1).getType()); Assert.assertEquals("typeC", assetTypes.get(2).getType()); } finally { - assets.forEach((asset) -> { assetService.deleteAsset(tenantId, asset.getId()); }); + assets.forEach((asset) -> { + assetService.deleteAsset(tenantId, asset.getId()); + }); } } @@ -267,10 +282,10 @@ public class AssetServiceTest extends AbstractServiceTest { @Test public void testFindAssetsByTenantId() { List assets = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("Asset"+i); + asset.setName("Asset" + i); asset.setType("default"); assets.add(assetService.saveAsset(asset)); } @@ -303,11 +318,11 @@ public class AssetServiceTest extends AbstractServiceTest { public void testFindAssetsByTenantIdAndName() { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -315,11 +330,11 @@ public class AssetServiceTest extends AbstractServiceTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i=0;i<17;i++) { + for (int i = 0; i < 17; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -381,11 +396,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title1 = "Asset title 1"; String type1 = "typeA"; List assetsType1 = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -394,11 +409,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title2 = "Asset title 2"; String type2 = "typeB"; List assetsType2 = new ArrayList<>(); - for (int i=0;i<17;i++) { + for (int i = 0; i < 17; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); @@ -464,10 +479,10 @@ public class AssetServiceTest extends AbstractServiceTest { CustomerId customerId = customer.getId(); List assets = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("Asset"+i); + asset.setName("Asset" + i); asset.setType("default"); asset = assetService.saveAsset(asset); assets.add(new AssetInfo(assetService.assignAssetToCustomer(tenantId, asset.getId(), customerId), customer.getTitle(), customer.isPublic(), "default")); @@ -508,11 +523,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i=0;i<17;i++) { + for (int i = 0; i < 17; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -521,11 +536,11 @@ public class AssetServiceTest extends AbstractServiceTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -596,11 +611,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title1 = "Asset title 1"; String type1 = "typeC"; List assetsType1 = new ArrayList<>(); - for (int i=0;i<17;i++) { + for (int i = 0; i < 17; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -610,11 +625,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title2 = "Asset title 2"; String type2 = "typeD"; List assetsType2 = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); @@ -848,4 +863,52 @@ public class AssetServiceTest extends AbstractServiceTest { ); } + @Test + public void testDeleteAssetIfReferencedInCalculatedField() { + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName("My asset"); + asset.setType("default"); + Asset savedAsset = assetService.saveAsset(asset); + + Asset assetWithCf = new Asset(); + assetWithCf.setTenantId(tenantId); + assetWithCf.setName("Asset with CF"); + assetWithCf.setType("default"); + Asset savedAssetWithCf = assetService.saveAsset(assetWithCf); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setName("Test CF"); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setEntityId(savedAssetWithCf.getId()); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + argument.setRefEntityId(savedAsset.getId()); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + + config.setArguments(Map.of("T", argument)); + + config.setExpression("T - (100 - H) / 5"); + + Output output = new Output(); + output.setName("output"); + output.setType(OutputType.TIME_SERIES); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> assetService.deleteAsset(tenantId, savedAsset.getId())) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't delete asset that has entity views or is referenced in calculated fields!"); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java new file mode 100644 index 0000000000..2985aa7620 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -0,0 +1,168 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service; + +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.exception.DataValidationException; + +import java.util.Map; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@DaoSqlTest +public class CalculatedFieldServiceTest extends AbstractServiceTest { + + @Autowired + private CalculatedFieldService calculatedFieldService; + @Autowired + private DeviceService deviceService; + + private ListeningExecutorService executor; + + @Before + public void before() { + executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(8, getClass())); + } + + @After + public void after() { + executor.shutdownNow(); + } + + @Test + public void testSaveCalculatedField() { + Device device = createTestDevice(); + CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + + assertThat(savedCalculatedField).isNotNull(); + assertThat(savedCalculatedField.getId()).isNotNull(); + assertThat(savedCalculatedField.getCreatedTime()).isGreaterThan(0); + assertThat(savedCalculatedField.getTenantId()).isEqualTo(calculatedField.getTenantId()); + assertThat(savedCalculatedField.getEntityId()).isEqualTo(calculatedField.getEntityId()); + assertThat(savedCalculatedField.getType()).isEqualTo(calculatedField.getType()); + assertThat(savedCalculatedField.getName()).isEqualTo(calculatedField.getName()); + assertThat(savedCalculatedField.getConfiguration()).isEqualTo(calculatedField.getConfiguration()); + assertThat(savedCalculatedField.getVersion()).isEqualTo(1L); + + savedCalculatedField.setName("Test CF"); + + CalculatedField updatedCalculatedField = calculatedFieldService.save(savedCalculatedField); + + assertThat(updatedCalculatedField.getName()).isEqualTo(savedCalculatedField.getName()); + assertThat(updatedCalculatedField.getVersion()).isEqualTo(savedCalculatedField.getVersion() + 1); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + } + + @Test + public void testSaveCalculatedFieldWithExistingName() { + Device device = createTestDevice(); + CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); + calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Calculated Field with such name is already in exists!"); + } + + @Test + public void testFindCalculatedFieldById() { + CalculatedField savedCalculatedField = saveValidCalculatedField(); + CalculatedField fetchedCalculatedField = calculatedFieldService.findById(tenantId, savedCalculatedField.getId()); + + assertThat(fetchedCalculatedField).isEqualTo(savedCalculatedField); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + } + + @Test + public void testDeleteCalculatedField() { + CalculatedField savedCalculatedField = saveValidCalculatedField(); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + + assertThat(calculatedFieldService.findById(tenantId, savedCalculatedField.getId())).isNull(); + } + + private CalculatedField saveValidCalculatedField() { + Device device = createTestDevice(); + CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); + return calculatedFieldService.save(calculatedField); + } + + private CalculatedField getCalculatedField(EntityId entityId, EntityId referencedEntityId) { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setEntityId(entityId); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); + return calculatedField; + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + argument.setRefEntityId(referencedEntityId); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + + config.setArguments(Map.of("T", argument)); + + config.setExpression("T - (100 - H) / 5"); + + Output output = new Output(); + output.setName("output"); + output.setType(OutputType.TIME_SERIES); + + config.setOutput(output); + + return config; + } + + private Device createTestDevice() { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Test"); + return deviceService.saveDevice(device); + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 3e1dbe08a8..daa10e72e9 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -30,14 +30,26 @@ import org.testcontainers.shaded.org.awaitility.Awaitility; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.exception.DataValidationException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -45,13 +57,17 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; +import static org.assertj.core.api.Assertions.assertThatThrownBy; @DaoSqlTest public class CustomerServiceTest extends AbstractServiceTest { @Autowired CustomerService customerService; + @Autowired + CalculatedFieldService calculatedFieldService; + @Autowired + AssetService assetService; static final int TIMEOUT = 30; @@ -343,4 +359,51 @@ public class CustomerServiceTest extends AbstractServiceTest { } } + @Test + public void testDeleteCustomerIfReferencedInCalculatedField() { + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle("My customer"); + Customer savedCustomer = customerService.saveCustomer(customer); + + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName("My asset"); + asset.setType("default"); + Asset savedAsset = assetService.saveAsset(asset); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setName("Test CF"); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setEntityId(savedAsset.getId()); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + argument.setRefEntityId(savedCustomer.getId()); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + + config.setArguments(Map.of("T", argument)); + + config.setExpression("T - (100 - H) / 5"); + + Output output = new Output(); + output.setName("output"); + output.setType(OutputType.TIME_SERIES); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> customerService.deleteCustomer(tenantId, savedCustomer.getId())) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't delete customer that is referenced in calculated fields!"); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index bf9c94dc89..bbbd48aa49 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -39,6 +39,14 @@ import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -50,6 +58,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceProfileService; @@ -64,6 +73,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; @@ -87,6 +97,8 @@ public class DeviceServiceTest extends AbstractServiceTest { @Autowired TenantProfileService tenantProfileService; @Autowired + private CalculatedFieldService calculatedFieldService; + @Autowired private PlatformTransactionManager platformTransactionManager; @SpyBean private DeviceCredentialsDataValidator validator; @@ -1198,4 +1210,43 @@ public class DeviceServiceTest extends AbstractServiceTest { ); } + @Test + public void testDeleteDeviceIfReferencedInCalculatedField() { + Device device = saveDevice(tenantId, "Test Device"); + Device deviceWithCf = saveDevice(tenantId, "Device with CF"); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setName("Test CF"); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setEntityId(deviceWithCf.getId()); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + argument.setRefEntityId(device.getId()); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + + config.setArguments(Map.of("T", argument)); + + config.setExpression("T - (100 - H) / 5"); + + Output output = new Output(); + output.setName("output"); + output.setType(OutputType.TIME_SERIES); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> deviceService.deleteDevice(tenantId, device.getId())) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't delete device that has entity views or is referenced in calculated fields!"); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java index 0e7797ce56..9503f4e23f 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java @@ -20,6 +20,7 @@ import org.junit.Assert; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.entity.EntityDaoService; import org.thingsboard.server.dao.entity.EntityServiceRegistry; import org.thingsboard.server.dao.rule.RuleChainService; @@ -44,4 +45,8 @@ public class EntityServiceRegistryTest extends AbstractServiceTest { Assert.assertTrue(entityServiceRegistry.getServiceByEntityType(EntityType.RULE_NODE) instanceof RuleChainService); } + @Test + public void givenCalculatedFieldLinkEntityType_whenGetServiceByEntityTypeCalled_thenReturnedCalculatedFieldService() { + Assert.assertTrue(entityServiceRegistry.getServiceByEntityType(EntityType.CALCULATED_FIELD_LINK) instanceof CalculatedFieldService); + } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java new file mode 100644 index 0000000000..43fb44431e --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.validator; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.cf.CalculatedFieldDao; +import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.usagerecord.DefaultApiLimitService; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.BDDMockito.given; + +@SpringBootTest(classes = CalculatedFieldDataValidator.class) +public class CalculatedFieldDataValidatorTest { + + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("7b5229e9-166e-41a9-a257-3b1dafad1b04")); + private final CalculatedFieldId CALCULATED_FIELD_ID = new CalculatedFieldId(UUID.fromString("060fbe45-fbb2-4549-abf3-f72a6be3cb9f")); + + @MockBean + private CalculatedFieldDao calculatedFieldDao; + @MockBean + private DefaultApiLimitService apiLimitService; + @SpyBean + private CalculatedFieldDataValidator validator; + + @Test + public void testUpdateNonExistingCalculatedField() { + CalculatedField calculatedField = new CalculatedField(CALCULATED_FIELD_ID); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("Test"); + + given(calculatedFieldDao.findById(TENANT_ID, CALCULATED_FIELD_ID.getId())).willReturn(null); + + assertThatThrownBy(() -> validator.validateUpdate(TENANT_ID, calculatedField)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't update non existing calculated field!"); + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java new file mode 100644 index 0000000000..c477498602 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.validator; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.cf.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.exception.DataValidationException; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.BDDMockito.given; + +@SpringBootTest(classes = CalculatedFieldLinkDataValidator.class) +public class CalculatedFieldLinkDataValidatorTest { + + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("2ba09d99-6143-43dc-b645-381fc0c43ebe")); + private final CalculatedFieldLinkId CALCULATED_FIELD_LINK_ID = new CalculatedFieldLinkId(UUID.fromString("a5609ef4-cb42-43ce-9b23-e090a4878d1c")); + + @MockBean + private CalculatedFieldLinkDao calculatedFieldLinkDao; + @SpyBean + private CalculatedFieldLinkDataValidator validator; + + @Test + public void testUpdateNonExistingCalculatedField() { + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(CALCULATED_FIELD_LINK_ID); + calculatedFieldLink.setCalculatedFieldId(new CalculatedFieldId(UUID.fromString("136477af-fd07-4498-b9c9-54fe50e82992"))); + + given(calculatedFieldLinkDao.findById(TENANT_ID, CALCULATED_FIELD_LINK_ID.getId())).willReturn(null); + + assertThatThrownBy(() -> validator.validateUpdate(TENANT_ID, calculatedFieldLink)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't update non existing calculated field link!"); + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java index e69c52056a..7e3ec8ba56 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java @@ -29,6 +29,8 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.junit4.SpringRunner; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.permission.QueryContext; import java.util.List; import java.util.UUID; @@ -48,7 +50,7 @@ import static org.mockito.Mockito.times; public class DefaultQueryLogComponentTest { private TenantId tenantId; - private QueryContext ctx; + private SqlQueryContext ctx; @SpyBean private DefaultQueryLogComponent queryLog; @@ -56,7 +58,7 @@ public class DefaultQueryLogComponentTest { @Before public void setUp() { tenantId = new TenantId(UUID.fromString("97275c1c-9cf2-4d25-a68d-933031158f84")); - ctx = new QueryContext(new QuerySecurityContext(tenantId, null, EntityType.ALARM)); + ctx = new SqlQueryContext(new QueryContext(tenantId, null, EntityType.ALARM)); } @Test diff --git a/dao/src/test/resources/application-test.properties b/dao/src/test/resources/application-test.properties index 1e11c77f9a..a1bb335ad0 100644 --- a/dao/src/test/resources/application-test.properties +++ b/dao/src/test/resources/application-test.properties @@ -145,6 +145,8 @@ sql.events.batch_threads=2 actors.system.tenant_dispatcher_pool_size=4 actors.system.device_dispatcher_pool_size=8 actors.system.rule_dispatcher_pool_size=12 +actors.system.cfm_dispatcher_pool_size=2 +actors.system.cfe_dispatcher_pool_size=2 transport.sessions.report_timeout=10000 queue.transport_api.request_poll_interval=5 queue.transport_api.response_poll_interval=5 diff --git a/docker/.env b/docker/.env index 37c9768296..71722247df 100644 --- a/docker/.env +++ b/docker/.env @@ -14,6 +14,8 @@ COAP_TRANSPORT_DOCKER_NAME=tb-coap-transport LWM2M_TRANSPORT_DOCKER_NAME=tb-lwm2m-transport SNMP_TRANSPORT_DOCKER_NAME=tb-snmp-transport TB_VC_EXECUTOR_DOCKER_NAME=tb-vc-executor +EDQS_DOCKER_NAME=tb-edqs +EDQS_ENABLED=false TB_VERSION=latest diff --git a/docker/compose-utils.sh b/docker/compose-utils.sh index 55615800b4..3862024786 100755 --- a/docker/compose-utils.sh +++ b/docker/compose-utils.sh @@ -128,6 +128,18 @@ function additionalStartupServices() { echo $ADDITIONAL_STARTUP_SERVICES } +function additionalComposeEdqsArgs() { + source .env + + if [ "$EDQS_ENABLED" = true ] + then + ADDITIONAL_COMPOSE_EDQS_ARGS="-f docker-compose.edqs.yml" + echo ADDITIONAL_COMPOSE_EDQS_ARGS + else + echo "" + fi +} + function permissionList() { PERMISSION_LIST=" 799 799 tb-node/log @@ -148,6 +160,12 @@ function permissionList() { " fi + if [ "$EDQS_ENABLED" = true ]; then + PERMISSION_LIST="$PERMISSION_LIST + 799 799 edqs/log + " + fi + CACHE="${CACHE:-redis}" case $CACHE in redis) diff --git a/docker/docker-compose.edqs.volumes.yml b/docker/docker-compose.edqs.volumes.yml new file mode 100644 index 0000000000..9d2ce946c8 --- /dev/null +++ b/docker/docker-compose.edqs.volumes.yml @@ -0,0 +1,30 @@ +# +# Copyright © 2016-2025 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '3.0' + +services: + tb-edqs-1: + volumes: + - tb-edqs-log-volume:/var/log/edqs + tb-edqs-2: + volumes: + - tb-edqs-log-volume:/var/log/edqs + +volumes: + tb-edqs-log-volume: + external: + name: ${TB_EDQS_LOG_VOLUME} diff --git a/docker/docker-compose.edqs.yml b/docker/docker-compose.edqs.yml new file mode 100644 index 0000000000..67e9c987e3 --- /dev/null +++ b/docker/docker-compose.edqs.yml @@ -0,0 +1,57 @@ +# +# Copyright © 2016-2025 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '3.0' + +services: + tb-core1: + env_file: + - tb-core-edqs.env + tb-core2: + env_file: + - tb-core-edqs.env + tb-rule-engine1: + env_file: + - tb-rule-engine-edqs.env + tb-rule-engine2: + env_file: + - tb-rule-engine-edqs.env + tb-edqs-1: + restart: always + image: "${DOCKER_REPO}/${EDQS_DOCKER_NAME}:${TB_VERSION}" + env_file: + - edqs.env + volumes: + - ./edqs/conf:/usr/share/edqs/conf + - ./edqs/log:/var/log/edqs + ports: + - "8080" + depends_on: + - zookeeper + - kafka + tb-edqs-2: + restart: always + image: "${DOCKER_REPO}/${EDQS_DOCKER_NAME}:${TB_VERSION}" + env_file: + - edqs.env + volumes: + - ./edqs/conf:/usr/share/edqs/conf + - ./edqs/log:/var/log/edqs + ports: + - "8080" + depends_on: + - zookeeper + - kafka diff --git a/docker/docker-install-tb.sh b/docker/docker-install-tb.sh index 25f089afb4..aa68a2252f 100755 --- a/docker/docker-install-tb.sh +++ b/docker/docker-install-tb.sh @@ -49,12 +49,15 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then COMPOSE_ARGS="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} + ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ up -d ${ADDITIONAL_STARTUP_SERVICES}" case $COMPOSE_VERSION in @@ -71,7 +74,8 @@ if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then fi COMPOSE_ARGS="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} + ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} \ tb-core1" diff --git a/docker/docker-remove-services.sh b/docker/docker-remove-services.sh index 6b36f4be08..3124119abc 100755 --- a/docker/docker-remove-services.sh +++ b/docker/docker-remove-services.sh @@ -29,8 +29,10 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + COMPOSE_ARGS="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ down -v" case $COMPOSE_VERSION in diff --git a/docker/docker-start-services.sh b/docker/docker-start-services.sh index 39dc57c1de..0d256abcf6 100755 --- a/docker/docker-start-services.sh +++ b/docker/docker-start-services.sh @@ -29,8 +29,10 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + COMPOSE_ARGS="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ up -d" case $COMPOSE_VERSION in diff --git a/docker/docker-stop-services.sh b/docker/docker-stop-services.sh index 670c44ca92..54386d10dd 100755 --- a/docker/docker-stop-services.sh +++ b/docker/docker-stop-services.sh @@ -29,8 +29,10 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + COMPOSE_ARGS="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} ${ADDITIONAL_COMPOSE_EDQS_ARGS}\ stop" case $COMPOSE_VERSION in diff --git a/docker/docker-update-service.sh b/docker/docker-update-service.sh index 7a77241e48..de1fe0a89a 100755 --- a/docker/docker-update-service.sh +++ b/docker/docker-update-service.sh @@ -27,12 +27,16 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + COMPOSE_ARGS_PULL="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} + ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ pull" COMPOSE_ARGS_BUILD="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} + ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ up -d --no-deps --build" case $COMPOSE_VERSION in diff --git a/docker/docker-upgrade-tb.sh b/docker/docker-upgrade-tb.sh index 5a34e29aa9..05293e475e 100755 --- a/docker/docker-upgrade-tb.sh +++ b/docker/docker-upgrade-tb.sh @@ -42,19 +42,24 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? COMPOSE_ARGS_PULL="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} + ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ pull \ tb-core1" COMPOSE_ARGS_UP="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} + ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ up -d ${ADDITIONAL_STARTUP_SERVICES}" COMPOSE_ARGS_RUN="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} + ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} \ tb-core1" diff --git a/docker/edqs.env b/docker/edqs.env new file mode 100644 index 0000000000..2c07d6e80d --- /dev/null +++ b/docker/edqs.env @@ -0,0 +1,7 @@ +ZOOKEEPER_ENABLED=true +ZOOKEEPER_URL=zookeeper:2181 +TB_KAFKA_SERVERS=kafka:9092 +HTTP_BIND_PORT=8080 + +METRICS_ENABLED=true +METRICS_ENDPOINTS_EXPOSE=prometheus diff --git a/docker/edqs/conf/edqs.conf b/docker/edqs/conf/edqs.conf new file mode 100644 index 0000000000..8c6b5d1826 --- /dev/null +++ b/docker/edqs/conf/edqs.conf @@ -0,0 +1,22 @@ +# +# Copyright © 2016-2025 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=/var/log/edqs/${TB_SERVICE_ID}-gc.log:time,uptime,level,tags:filecount=10,filesize=10M" +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError" +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" +export LOG_FILENAME=tb-edqs.out +export LOADER_PATH=/usr/share/edqs/conf diff --git a/docker/edqs/conf/logback.xml b/docker/edqs/conf/logback.xml new file mode 100644 index 0000000000..40481a8c35 --- /dev/null +++ b/docker/edqs/conf/logback.xml @@ -0,0 +1,52 @@ + + + + + + + /var/log/edqs/${TB_SERVICE_ID}/tb-edqs.log + + /var/log/edqs/tb-edqs.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + + diff --git a/docker/monitoring/grafana/provisioning/dashboards/edqs_entities.json b/docker/monitoring/grafana/provisioning/dashboards/edqs_entities.json new file mode 100644 index 0000000000..3e913d4856 --- /dev/null +++ b/docker/monitoring/grafana/provisioning/dashboards/edqs_entities.json @@ -0,0 +1,161 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 6, + "iteration": 1737564772936, + "links": [], + "liveNow": false, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "9BonzvTSz" + }, + "exemplar": true, + "expr": "sum by (objectType) (edqs_object_count{tenantId=~\"$tenantId\"})", + "interval": "", + "legendFormat": "{{objectType}}", + "refId": "A" + } + ], + "title": "EDQS object count", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "definition": "label_values(edqs_object_count, tenantId)", + "hide": 0, + "includeAll": true, + "label": "Tenant", + "multi": true, + "name": "tenantId", + "options": [], + "query": { + "query": "label_values(edqs_object_count, tenantId)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "EDQS", + "uid": "mK5A_DdHk", + "version": 9, + "weekStart": "" +} \ No newline at end of file diff --git a/docker/tb-core-edqs.env b/docker/tb-core-edqs.env new file mode 100644 index 0000000000..45472ec907 --- /dev/null +++ b/docker/tb-core-edqs.env @@ -0,0 +1,5 @@ +# ThingsBoard server configuration with enabled EDQS synchronization + +TB_EDQS_MODE=remote +TB_EDQS_SYNC_ENABLED=true +TB_EDQS_API_SUPPORTED=true diff --git a/docker/tb-rule-engine-edqs.env b/docker/tb-rule-engine-edqs.env new file mode 100644 index 0000000000..82395ddcfe --- /dev/null +++ b/docker/tb-rule-engine-edqs.env @@ -0,0 +1,3 @@ +# ThingsBoard server configuration with enabled EDQS synchronization + +TB_EDQS_SYNC_ENABLED=true diff --git a/edqs/pom.xml b/edqs/pom.xml new file mode 100644 index 0000000000..07b25102d4 --- /dev/null +++ b/edqs/pom.xml @@ -0,0 +1,213 @@ + + + 4.0.0 + + org.thingsboard + 4.0.0-SNAPSHOT + thingsboard + + edqs + jar + + ThingsBoard Entity Data Query Service Application + https://thingsboard.io + + + UTF-8 + ${basedir}/.. + java + false + process-resources + package + edqs + ${project.build.directory}/windows + true + ThingsBoard Entity Data Query Service + org.thingsboard.server.edqs.ThingsboardEdqsApplication + + + + + org.thingsboard.common + edqs + + + org.slf4j + slf4j-api + + + org.slf4j + log4j-over-slf4j + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + org.apache.curator + curator-recipes + + + com.google.protobuf + protobuf-java + + + io.grpc + grpc-protobuf + + + io.grpc + grpc-stub + + + + com.sun.winsw + winsw + bin + exe + provided + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.awaitility + awaitility + test + + + org.assertj + assertj-core + test + + + + + ${pkg.name}-${project.version} + + + ${project.basedir}/src/main/resources + true + + edqs.yml + + + + ${project.basedir}/src/main/resources + false + + edqs.yml + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + + + thingsboard + + + **/nosql/*Test.java + + + **/*Test.java + **/*TestSuite.java + + + + + org.apache.maven.plugins + maven-resources-plugin + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-winsw-service + package + + + + + org.apache.maven.plugins + maven-jar-plugin + + + org.springframework.boot + spring-boot-maven-plugin + + + org.thingsboard + gradle-maven-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.maven.plugins + maven-install-plugin + + + org.xolstice.maven.plugins + protobuf-maven-plugin + + + org.codehaus.mojo + build-helper-maven-plugin + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + + + jenkins + Jenkins Repository + https://repo.jenkins-ci.org/releases + + false + + + + diff --git a/edqs/src/main/conf/edqs.conf b/edqs/src/main/conf/edqs.conf new file mode 100644 index 0000000000..3f96fd590d --- /dev/null +++ b/edqs/src/main/conf/edqs.conf @@ -0,0 +1,22 @@ +# +# Copyright © 2016-2025 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=@pkg.logFolder@/gc.log:time,uptime,level,tags:filecount=10,filesize=10M" +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError" +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" +export LOG_FILENAME=${pkg.name}.out +export LOADER_PATH=${pkg.installFolder}/conf diff --git a/edqs/src/main/conf/logback.xml b/edqs/src/main/conf/logback.xml new file mode 100644 index 0000000000..850a28b212 --- /dev/null +++ b/edqs/src/main/conf/logback.xml @@ -0,0 +1,49 @@ + + + + + + + ${pkg.logFolder}/${pkg.name}.log + + ${pkg.logFolder}/${pkg.name}.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + diff --git a/edqs/src/main/java/org/thingsboard/server/edqs/DummyQueueRoutingInfoService.java b/edqs/src/main/java/org/thingsboard/server/edqs/DummyQueueRoutingInfoService.java new file mode 100644 index 0000000000..1f1152af68 --- /dev/null +++ b/edqs/src/main/java/org/thingsboard/server/edqs/DummyQueueRoutingInfoService.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.discovery.QueueRoutingInfo; +import org.thingsboard.server.queue.discovery.QueueRoutingInfoService; + +import java.util.Collections; +import java.util.List; + +@Service +public class DummyQueueRoutingInfoService implements QueueRoutingInfoService { + + @Override + public List getAllQueuesRoutingInfo() { + return Collections.emptyList(); + } + +} diff --git a/edqs/src/main/java/org/thingsboard/server/edqs/DummyTenantRoutingInfoService.java b/edqs/src/main/java/org/thingsboard/server/edqs/DummyTenantRoutingInfoService.java new file mode 100644 index 0000000000..4e16e5e16a --- /dev/null +++ b/edqs/src/main/java/org/thingsboard/server/edqs/DummyTenantRoutingInfoService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.discovery.TenantRoutingInfo; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; + +@Service +public class DummyTenantRoutingInfoService implements TenantRoutingInfoService { + @Override + public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { + return null; + } + +} diff --git a/edqs/src/main/java/org/thingsboard/server/edqs/EdqsController.java b/edqs/src/main/java/org/thingsboard/server/edqs/EdqsController.java new file mode 100644 index 0000000000..4d0858ad5a --- /dev/null +++ b/edqs/src/main/java/org/thingsboard/server/edqs/EdqsController.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs; + +import lombok.RequiredArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.edqs.state.EdqsStateService; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/api/edqs") +public class EdqsController { + + private final EdqsStateService edqsStateService; + + @GetMapping("/ready") + public ResponseEntity isReady() { + if (edqsStateService.isReady()) { + return ResponseEntity.ok().build(); + } else { + return ResponseEntity.badRequest().build(); + } + } + +} diff --git a/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java b/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java new file mode 100644 index 0000000000..00b3fdbd26 --- /dev/null +++ b/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.annotation.EnableScheduling; + +import java.util.Arrays; + +@SpringBootConfiguration +@EnableAsync +@EnableScheduling +@EnableAutoConfiguration +@ComponentScan({"org.thingsboard.server.edqs", "org.thingsboard.server.queue.edqs", "org.thingsboard.server.queue.discovery", "org.thingsboard.server.queue.kafka", + "org.thingsboard.server.queue.settings", "org.thingsboard.server.queue.environment", "org.thingsboard.server.common.stats"}) +@Slf4j +public class ThingsboardEdqsApplication { + + private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; + private static final String DEFAULT_SPRING_CONFIG_PARAM = SPRING_CONFIG_NAME_KEY + "=" + "edqs"; + + public static void main(String[] args) { + SpringApplication.run(ThingsboardEdqsApplication.class, updateArguments(args)); + } + + private static String[] updateArguments(String[] args) { + if (Arrays.stream(args).noneMatch(arg -> arg.startsWith(SPRING_CONFIG_NAME_KEY))) { + String[] modifiedArgs = new String[args.length + 1]; + System.arraycopy(args, 0, modifiedArgs, 0, args.length); + modifiedArgs[args.length] = DEFAULT_SPRING_CONFIG_PARAM; + return modifiedArgs; + } + return args; + } + +} diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml new file mode 100644 index 0000000000..1d7f111e9a --- /dev/null +++ b/edqs/src/main/resources/edqs.yml @@ -0,0 +1,201 @@ +# +# Copyright © 2016-2025 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +server: + # Server bind-address + address: "${HTTP_BIND_ADDRESS:0.0.0.0}" + # Server bind port + port: "${HTTP_BIND_PORT:8080}" + +# Application info parameters +app: + # Application version + version: "@project.version@" + +# Zookeeper connection parameters +zk: + # Enable/disable zookeeper discovery service. + enabled: "${ZOOKEEPER_ENABLED:true}" + # Zookeeper connect string + url: "${ZOOKEEPER_URL:localhost:2181}" + # Zookeeper retry interval in milliseconds + retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" + # Zookeeper connection timeout in milliseconds + connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" + # Zookeeper session timeout in milliseconds + session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" + # Name of the directory in zookeeper 'filesystem' + zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" + # The recalculate_delay property is recommended in a microservices architecture setup for rule-engine services. + # This property provides a pause to ensure that when a rule-engine service is restarted, other nodes don't immediately attempt to recalculate their partitions. + # The delay is recommended because the initialization of rule chain actors is time-consuming. Avoiding unnecessary recalculations during a restart can enhance system performance and stability. + recalculate_delay: "${ZOOKEEPER_RECALCULATE_DELAY_MS:0}" + +spring: + main: + allow-circular-references: "true" # Spring Boot configuration property that controls whether circular dependencies between beans are allowed. + +# Queue configuration parameters +queue: + type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) + prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). + edqs: + # Number of partitions for EDQS topics + partitions: "${TB_EDQS_PARTITIONS:12}" + # EDQS partitioning strategy: tenant (partitions are resolved and distributed by tenant id) or none (partitions are resolved by message key; each instance has all the partitions) + partitioning_strategy: "${TB_EDQS_PARTITIONING_STRATEGY:tenant}" + # EDQS requests topic + requests_topic: "${TB_EDQS_REQUESTS_TOPIC:edqs.requests}" + # EDQS responses topic + responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}" + # Poll interval for EDQS topics + poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}" + # Maximum amount of pending requests to EDQS + max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}" + # Maximum timeout for requests to EDQS + max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:20000}" + stats: + # Enable/disable statistics for EDQS + enabled: "${TB_EDQS_STATS_ENABLED:true}" + # Statistics printing interval for EDQS + print-interval-ms: "${TB_EDQS_STATS_PRINT_INTERVAL_MS:300000}" + + kafka: + # Kafka Bootstrap nodes in "host:port" format + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + ssl: + # Enable/Disable SSL Kafka communication + enabled: "${TB_KAFKA_SSL_ENABLED:false}" + # The location of the trust store file + truststore.location: "${TB_KAFKA_SSL_TRUSTSTORE_LOCATION:}" + # The password of trust store file if specified + truststore.password: "${TB_KAFKA_SSL_TRUSTSTORE_PASSWORD:}" + # The location of the key store file. This is optional for the client and can be used for two-way authentication for the client + keystore.location: "${TB_KAFKA_SSL_KEYSTORE_LOCATION:}" + # The store password for the key store file. This is optional for the client and only needed if ‘ssl.keystore.location’ is configured. Key store password is not supported for PEM format + keystore.password: "${TB_KAFKA_SSL_KEYSTORE_PASSWORD:}" + # The password of the private key in the key store file or the PEM key specified in ‘keystore.key’ + key.password: "${TB_KAFKA_SSL_KEY_PASSWORD:}" + # The number of acknowledgments the producer requires the leader to have received before considering a request complete. This controls the durability of records that are sent. The following settings are allowed:0, 1 and all + acks: "${TB_KAFKA_ACKS:all}" + # Number of retries. Resend any record whose send fails with a potentially transient error + retries: "${TB_KAFKA_RETRIES:1}" + # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values none or gzip + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + # Default batch size. This setting gives the upper bound of the batch size to be sent + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + # This variable creates a small amount of artificial delay—that is, rather than immediately sending out a record + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + # The maximum size of a request in bytes. This setting will limit the number of record batches the producer will send in a single request to avoid sending huge requests + max.request.size: "${TB_KAFKA_MAX_REQUEST_SIZE:1048576}" + # The maximum number of unacknowledged requests the client will send on a single connection before blocking + max.in.flight.requests.per.connection: "${TB_KAFKA_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION:5}" + # The total bytes of memory the producer can use to buffer records waiting to be sent to the server + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + # The multiple copies of data over the multiple brokers of Kafka + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + # The maximum delay between invocations of poll() method when using consumer group management. This places an upper bound on the amount of time that the consumer can be idle before fetching more records + max_poll_interval_ms: "${TB_QUEUE_KAFKA_MAX_POLL_INTERVAL_MS:300000}" + # The maximum number of records returned in a single call of poll() method + max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" + # The maximum amount of data per-partition the server will return. Records are fetched in batches by the consumer + max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" + # The maximum amount of data the server will return. Records are fetched in batches by the consumer + fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" + request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none + # Enable/Disable using of Confluent Cloud + use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" + confluent: + # The endpoint identification algorithm used by clients to validate server hostname. The default value is https + ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" + # The mechanism used to authenticate Schema Registry requests. SASL/PLAIN should only be used with TLS/SSL as a transport layer to ensure that clear passwords are not transmitted on the wire without encryption + sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}" + # Using JAAS Configuration for specifying multiple SASL mechanisms on a broker + sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" + # Protocol used to communicate with brokers. Valid values are: PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL + security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" + # Key-value properties for Kafka consumer per specific topic, e.g. tb_ota_package is a topic name for ota, tb_rule_engine.sq is a topic name for default SequentialByOriginator queue. + # Check TB_QUEUE_CORE_OTA_TOPIC and TB_QUEUE_RE_SQ_TOPIC params + consumer-properties-per-topic: + edqs.events: + # Key-value properties for Kafka consumer for edqs.events topic + - key: max.poll.records + # Max poll records for edqs.events topic + value: "${TB_QUEUE_KAFKA_EDQS_EVENTS_MAX_POLL_RECORDS:512}" + edqs.state: + # Key-value properties for Kafka consumer for edqs.state topic + - key: max.poll.records + # Max poll records for edqs.state topic + value: "${TB_QUEUE_KAFKA_EDQS_STATE_MAX_POLL_RECORDS:512}" + + other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" + other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside + # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + # value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) + # - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + # value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) + topic-properties: + # Kafka properties for EDQS events topics. Partitions number must be the same as queue.edqs.partitions + edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1}" + # Kafka properties for EDQS requests topic (default: 3 minutes retention). Partitions number must be the same as queue.edqs.partitions + edqs-requests: "${TB_QUEUE_KAFKA_EDQS_REQUESTS_TOPIC_PROPERTIES:retention.ms:180000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}" + # Kafka properties for EDQS state topic (infinite retention, compaction). Partitions number must be the same as queue.edqs.partitions + edqs-state: "${TB_QUEUE_KAFKA_EDQS_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1;cleanup.policy:compact}" + consumer-stats: + # Prints lag between consumer group offset and last messages offset in Kafka topics + enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" + # Statistics printing interval for Kafka's consumer-groups stats + print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}" + # Time to wait for the stats-loading requests to Kafka to finish + kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 + +# General service parameters +service: + type: "${TB_SERVICE_TYPE:edqs}" + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" + edqs: + # EDQS instances with the same label will share the same list of partitions + label: "${TB_EDQS_LABEL:}" + +# Metrics parameters +metrics: + # Enable/disable actuator metrics. + enabled: "${METRICS_ENABLED:false}" + timer: + # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). + percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" + system_info: + # Persist frequency of system info (CPU, memory usage, etc.) in seconds + persist_frequency: "${METRICS_SYSTEM_INFO_PERSIST_FREQUENCY_SECONDS:60}" + # TTL in days for system info timeseries + ttl: "${METRICS_SYSTEM_INFO_TTL_DAYS:7}" + +# General management parameters +management: + endpoints: + web: + exposure: + # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). + include: '${METRICS_ENDPOINTS_EXPOSE:info}' + health: + elasticsearch: + # Enable the org.springframework.boot.actuate.elasticsearch.ElasticsearchRestClientHealthIndicator.doHealthCheck + enabled: "false" diff --git a/edqs/src/main/resources/logback.xml b/edqs/src/main/resources/logback.xml new file mode 100644 index 0000000000..5a6e9e3e24 --- /dev/null +++ b/edqs/src/main/resources/logback.xml @@ -0,0 +1,38 @@ + + + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + + + diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java new file mode 100644 index 0000000000..01a7495148 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java @@ -0,0 +1,252 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Before; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.TestExecutionListeners; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; +import org.springframework.test.context.support.DirtiesContextTestExecutionListener; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.util.EdqsConverter; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +@RunWith(SpringRunner.class) +@Configuration +@ComponentScan({"org.thingsboard.server.edqs.repo", "org.thingsboard.server.edqs.util"}) +@EntityScan("org.thingsboard.server.edqs") +@TestPropertySource(locations = {"classpath:edqs-test.properties"}) +@TestExecutionListeners({ + DependencyInjectionTestExecutionListener.class, + DirtiesContextTestExecutionListener.class}) +public abstract class AbstractEDQTest { + + @Autowired + protected DefaultEdqsRepository repository; + @Autowired + protected EdqsConverter edqsConverter; + + protected final TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); + protected final CustomerId customerId = new CustomerId(UUID.randomUUID()); + + protected final UUID defaultAssetProfileId = UUID.randomUUID(); + protected final UUID defaultDeviceProfileId = UUID.randomUUID(); + + @Before + public final void before() { + AssetProfile ap = new AssetProfile(new AssetProfileId(defaultAssetProfileId)); + ap.setName("default"); + ap.setDefault(true); + addOrUpdate(EntityType.ASSET_PROFILE, ap); + + DeviceProfile dp = new DeviceProfile(new DeviceProfileId(defaultDeviceProfileId)); + dp.setName("default"); + dp.setDefault(true); + dp.setType(DeviceProfileType.DEFAULT); + addOrUpdate(EntityType.DEVICE_PROFILE, dp); + + createCustomer(customerId.getId(), null, "Customer A"); + } + + @After + public final void after() { + repository.clear(); + } + + protected void createCustomer(UUID id, UUID parentCustomerId, String title) { + Customer entity = new Customer(); + entity.setId(new CustomerId(id)); + entity.setTitle(title); + addOrUpdate(EntityType.CUSTOMER, entity); + } + + + protected UUID createDevice(String name) { + return createDevice(null, defaultDeviceProfileId, name); + } + + protected UUID createDevice(CustomerId customerId, String name) { + return createDevice(customerId.getId(), defaultDeviceProfileId, name); + } + + protected UUID createDevice(UUID customerId, UUID profileId, String name) { + UUID entityId = UUID.randomUUID(); + Device entity = new Device(); + entity.setId(new DeviceId(entityId)); + if (profileId != null) { + entity.setDeviceProfileId(new DeviceProfileId(profileId)); + } + if (customerId != null) { + entity.setCustomerId(new CustomerId(customerId)); + } + entity.setName(name); + addOrUpdate(EntityType.DEVICE, entity); + return entityId; + } + + protected UUID createDashboard(String name) { + UUID entityId = UUID.randomUUID(); + Dashboard entity = new Dashboard(); + entity.setId(new DashboardId(entityId)); + entity.setTitle(name); + addOrUpdate(EntityType.DEVICE, entity); + return entityId; + } + + protected UUID createView(String name) { + return createView(null, "default", name); + } + + protected UUID createView(CustomerId customerId, String name) { + return createView(customerId.getId(), "default", name); + } + + protected UUID createView(UUID customerId, String type, String name) { + UUID entityId = UUID.randomUUID(); + EntityView entity = new EntityView(); + entity.setId(new EntityViewId(entityId)); + entity.setType(type); + if (customerId != null) { + entity.setCustomerId(new CustomerId(customerId)); + } + entity.setName(name); + addOrUpdate(EntityType.ENTITY_VIEW, entity); + return entityId; + } + + protected UUID createEdge(String name) { + return createEdge(null, "default", name); + } + + protected UUID createEdge(CustomerId customerId, String name) { + return createEdge(customerId.getId(), "default", name); + } + + protected UUID createEdge(UUID customerId, String type, String name) { + UUID id = UUID.randomUUID(); + Edge edge = new Edge(); + edge.setId(new EdgeId(id)); + edge.setTenantId(tenantId); + if (customerId != null) { + edge.setCustomerId(new CustomerId(customerId)); + } + edge.setType(type); + edge.setName(name); + edge.setCreatedTime(42L); + addOrUpdate(EntityType.EDGE, edge); + return id; + } + + + protected UUID createAsset(String name) { + return createAsset(null, defaultAssetProfileId, name); + } + + protected UUID createAsset(UUID customerId, String name) { + return createAsset(customerId, defaultAssetProfileId, name); + } + + protected UUID createAsset(UUID customerId, UUID profileId, String name) { + UUID entityId = UUID.randomUUID(); + Asset entity = new Asset(); + entity.setId(new AssetId(entityId)); + if (profileId != null) { + entity.setAssetProfileId(new AssetProfileId(profileId)); + } + if (customerId != null) { + entity.setCustomerId(new CustomerId(customerId)); + } + entity.setName(name); + addOrUpdate(EntityType.ASSET, entity); + return entityId; + } + + protected void createRelation(EntityType fromType, UUID fromId, EntityType toType, UUID toId, String type) { + createRelation(fromType, fromId, toType, toId, RelationTypeGroup.COMMON, type); + } + + protected void createRelation(EntityType fromType, UUID fromId, EntityType toType, UUID toId, RelationTypeGroup group, String type) { + addOrUpdate(new EntityRelation(EntityIdFactory.getByTypeAndUuid(fromType, fromId), EntityIdFactory.getByTypeAndUuid(toType, toId), type, group)); + } + + + protected boolean checkContains(PageData data, UUID entityId) { + return data.getData().stream().anyMatch(r -> r.getEntityId().getId().equals(entityId)); + } + + protected List createStringKeyFilters(String key, EntityKeyType keyType, StringFilterPredicate.StringOperation operation, String value) { + KeyFilter filter = new KeyFilter(); + filter.setKey(new EntityKey(keyType, key)); + filter.setValueType(EntityKeyValueType.STRING); + StringFilterPredicate predicate = new StringFilterPredicate(); + predicate.setValue(FilterPredicateValue.fromString(value)); + predicate.setOperation(operation); + predicate.setIgnoreCase(true); + filter.setPredicate(predicate); + return Collections.singletonList(filter); + } + + protected void addOrUpdate(EntityType entityType, Object entity) { + addOrUpdate(EdqsConverter.toEntity(entityType, entity)); + } + + protected void addOrUpdate(EdqsObject edqsObject) { + byte[] serialized = edqsConverter.serialize(edqsObject.type(), edqsObject); + edqsObject = edqsConverter.deserialize(edqsObject.type(), serialized); + repository.get(tenantId).addOrUpdate(edqsObject); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/ApiUsageStateFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/ApiUsageStateFilterTest.java new file mode 100644 index 0000000000..e9470ca0b3 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/ApiUsageStateFilterTest.java @@ -0,0 +1,105 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.ApiUsageStateValue; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.id.ApiUsageStateId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.query.ApiUsageStateFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; + +import java.util.Arrays; +import java.util.UUID; + +public class ApiUsageStateFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + Tenant entity = new Tenant(); + entity.setId(tenantId); + entity.setTitle("test tenant"); + addOrUpdate(EntityType.TENANT, entity); + } + + @After + public void tearDown() { + } + + @Test + public void testFindCustomerApiUsageState() { + UUID customerId = UUID.randomUUID(); + createCustomer(customerId, null, "Customer A"); + + ApiUsageState apiUsageState = buildApiUsageState(customerId); + addOrUpdate(EntityType.API_USAGE_STATE, apiUsageState); + + var result = repository.findEntityDataByQuery(tenantId, null, getEntityDataQuery(new CustomerId(customerId)), false); + + Assert.assertEquals(1, result.getTotalElements()); + var customer = result.getData().get(0); + Assert.assertEquals("Customer A", customer.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + } + + private ApiUsageState buildApiUsageState(UUID customerId) { + ApiUsageState apiUsageState = new ApiUsageState(); + apiUsageState.setId(new ApiUsageStateId(UUID.randomUUID())); + apiUsageState.setTenantId(tenantId); + apiUsageState.setEntityId(new CustomerId(customerId)); + apiUsageState.setTransportState(ApiUsageStateValue.ENABLED); + apiUsageState.setReExecState(ApiUsageStateValue.ENABLED); + apiUsageState.setJsExecState(ApiUsageStateValue.ENABLED); + apiUsageState.setTbelExecState(ApiUsageStateValue.ENABLED); + apiUsageState.setDbStorageState(ApiUsageStateValue.ENABLED); + apiUsageState.setSmsExecState(ApiUsageStateValue.ENABLED); + apiUsageState.setEmailExecState(ApiUsageStateValue.ENABLED); + apiUsageState.setAlarmExecState(ApiUsageStateValue.ENABLED); + return apiUsageState; + } + + private static EntityDataQuery getEntityDataQuery(CustomerId customerId) { + ApiUsageStateFilter filter = new ApiUsageStateFilter(); + filter.setCustomerId(customerId); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "name"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); + predicate.setValue(new FilterPredicateValue<>("Customer A")); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + + return new EntityDataQuery(filter, pageLink, entityFields, null, Arrays.asList(nameFilter)); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetSearchQueryFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetSearchQueryFilterTest.java new file mode 100644 index 0000000000..1f90babf01 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetSearchQueryFilterTest.java @@ -0,0 +1,147 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +public class AssetSearchQueryFilterTest extends AbstractEDQTest { + private final AssetProfileId assetProfileId = new AssetProfileId(UUID.randomUUID()); + + @Before + public void setUp() { + } + + @Test + public void testFindTenantAssets() { + AssetProfile assetProfile = new AssetProfile(assetProfileId); + assetProfile.setName("Office"); + assetProfile.setDefault(false); + addOrUpdate(EntityType.ASSET_PROFILE, assetProfile); + + UUID root = createAsset(null, assetProfileId.getId(), "root"); + UUID asset1 = createAsset(null, assetProfileId.getId(), "A1"); + UUID asset2 = createAsset(null, assetProfileId.getId(), "A2"); + + createRelation(EntityType.ASSET, root, EntityType.ASSET, asset1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + + // find all assets of root asset + PageData relationsResult = findData(null, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("Office")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + Assert.assertTrue(checkContains(relationsResult, asset2)); + + // find all assets with max level = 1 + relationsResult = findData(null, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("Office")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + + // find all assets with asset type = default + relationsResult = findData(null, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("default")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all assets last level only, level = 2 + relationsResult = findData(null, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 2, true, Arrays.asList("Office")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset2)); + + // find all assets last level only, level = 1 + relationsResult = findData(null, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 1, true, Arrays.asList("Office")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + } + + @Test + public void testFindCustomerAssets() { + AssetProfile assetProfile = new AssetProfile(assetProfileId); + assetProfile.setName("Office"); + assetProfile.setDefault(false); + addOrUpdate(EntityType.ASSET_PROFILE, assetProfile); + + UUID root = createAsset(customerId.getId(), assetProfileId.getId(), "root"); + UUID asset1 = createAsset(customerId.getId(), assetProfileId.getId(), "A1"); + UUID asset2 = createAsset(customerId.getId(), assetProfileId.getId(), "A2"); + UUID asset3 = createAsset(customerId.getId(), defaultAssetProfileId, "A3"); + + createRelation(EntityType.ASSET, root, EntityType.ASSET, asset1, "Contains"); + createRelation(EntityType.ASSET, root, EntityType.ASSET, asset3, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + + // find all assets of root asset with profile "Office" + PageData relationsResult = findData(customerId, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("Office")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + Assert.assertTrue(checkContains(relationsResult, asset2)); + + // find all assets of root asset with profile "Office" and "default" + relationsResult = findData(customerId, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("Office", "default")); + Assert.assertEquals(3, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + Assert.assertTrue(checkContains(relationsResult, asset2)); + Assert.assertTrue(checkContains(relationsResult, asset3)); + + // find all assets with other customer + relationsResult = findData(new CustomerId(UUID.randomUUID()), new AssetId(root), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("Office")); + Assert.assertEquals(0, relationsResult.getData().size()); + } + + + private PageData findData(CustomerId customerId, EntityId rootId, + EntitySearchDirection direction, String relationType, int maxLevel, boolean lastLevelOnly, List assetTypes) { + AssetSearchQueryFilter filter = new AssetSearchQueryFilter(); + filter.setRootEntity(rootId); + filter.setDirection(direction); + filter.setRelationType(relationType); + filter.setAssetTypes(assetTypes); + filter.setFetchLastLevelOnly(lastLevelOnly); + filter.setMaxLevel(maxLevel); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "A"); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + return repository.findEntityDataByQuery(tenantId, customerId, query, false); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetTypeFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetTypeFilterTest.java new file mode 100644 index 0000000000..0961a7c12c --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetTypeFilterTest.java @@ -0,0 +1,185 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.AssetTypeFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class AssetTypeFilterTest extends AbstractEDQTest { + + private final AssetProfileId assetProfileId = new AssetProfileId(UUID.randomUUID()); + private final AssetProfileId assetProfileId2 = new AssetProfileId(UUID.randomUUID()); + private Asset asset; + private Asset asset2; + private Asset asset3; + + @Before + public void setUp() { + AssetProfile assetProfile = new AssetProfile(assetProfileId); + assetProfile.setName("Office"); + assetProfile.setDefault(false); + addOrUpdate(EntityType.ASSET_PROFILE, assetProfile); + + AssetProfile assetProfile2 = new AssetProfile(assetProfileId2); + assetProfile2.setName("Street"); + assetProfile2.setDefault(false); + addOrUpdate(EntityType.ASSET_PROFILE, assetProfile2); + + asset = buildAsset(assetProfileId, "Office 1"); + asset2 = buildAsset(assetProfileId, "Office 2"); + asset3 = buildAsset(assetProfileId2, "Abbey Road"); + + addOrUpdate(EntityType.ASSET, asset); + addOrUpdate(EntityType.ASSET, asset2); + addOrUpdate(EntityType.ASSET, asset3); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantAsset() { + // find asset with type "Office" + var result = repository.findEntityDataByQuery(tenantId, null, getAssetTypeQuery(Collections.singletonList("Office"), null, null), false); + + Assert.assertEquals(2, result.getTotalElements()); + var first = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("Office 1")).findAny(); + assertThat(first).isPresent(); + assertThat(first.get().getEntityId()).isEqualTo(asset.getId()); + assertThat(first.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(asset.getCreatedTime())); + + // find asset with type "Office" and "Street" + result = repository.findEntityDataByQuery(tenantId, null, getAssetTypeQuery(List.of("Office", "Street"), null, null), false); + + Assert.assertEquals(3, result.getTotalElements()); + var third = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("Abbey Road")).findAny(); + assertThat(third).isPresent(); + assertThat(third.get().getEntityId()).isEqualTo(asset3.getId()); + assertThat(third.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(asset.getCreatedTime())); + + // find asset with type "Supermarket" + result = repository.findEntityDataByQuery(tenantId, null, getAssetTypeQuery(List.of("Supermarket"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find asset with name "%Office%" + result = repository.findEntityDataByQuery(tenantId, null, getAssetTypeQuery(List.of("Office"), "%Office%", null), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find asset with name "Office 1" + result = repository.findEntityDataByQuery(tenantId, null, getAssetTypeQuery(List.of("Office"), "Office 1", null), false); + Assert.assertEquals(1, result.getTotalElements()); + + // find asset with name "%Super%" + result = repository.findEntityDataByQuery(tenantId, null, getAssetTypeQuery(List.of("Office"), "%Super%", null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find asset with key filter: name contains "Office" + KeyFilter containsNameFilter = getAssetNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "office", true); + result = repository.findEntityDataByQuery(tenantId, null, getAssetTypeQuery(List.of("Office"), null, Arrays.asList(containsNameFilter)), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find asset with key filter: name starts with "office" and matches case + KeyFilter startsWithNameFilter = getAssetNameKeyFilter(StringFilterPredicate.StringOperation.STARTS_WITH, "office", false); + result = repository.findEntityDataByQuery(tenantId, null, getAssetTypeQuery(List.of("Office"), null, Arrays.asList(startsWithNameFilter)), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerAsset() { + addOrUpdate(EntityType.ASSET, asset); + addOrUpdate(new LatestTsKv(asset.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, getAssetTypeQuery(List.of("Office"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + asset.setCustomerId(customerId); + addOrUpdate(EntityType.ASSET, asset); + + result = repository.findEntityDataByQuery(tenantId, customerId, getAssetTypeQuery(List.of("Office"), null, null), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(asset.getId(), first.getEntityId()); + Assert.assertEquals("Office 1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + + result = repository.findEntityDataByQuery(tenantId, customerId, getAssetTypeQuery(List.of("Supermarket"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + private Asset buildAsset(AssetProfileId assetProfileId, String assetName) { + Asset asset = new Asset(); + asset.setId(new AssetId(UUID.randomUUID())); + asset.setTenantId(tenantId); + asset.setAssetProfileId(assetProfileId); + asset.setName(assetName); + asset.setCreatedTime(42L); + return asset; + } + + private static EntityDataQuery getAssetTypeQuery(List assetTypes, String assetNameRegex, List keyFilters) { + AssetTypeFilter filter = new AssetTypeFilter(); + filter.setAssetTypes(assetTypes); + filter.setAssetNameFilter(assetNameRegex); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + } + + private static KeyFilter getAssetNameKeyFilter(StringFilterPredicate.StringOperation operation, String predicateValue, boolean ignoreCase) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(ignoreCase); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(predicateValue)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceSearchQueryFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceSearchQueryFilterTest.java new file mode 100644 index 0000000000..b3f15c2f61 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceSearchQueryFilterTest.java @@ -0,0 +1,150 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.DeviceSearchQueryFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +public class DeviceSearchQueryFilterTest extends AbstractEDQTest { + private final DeviceProfileId deviceProfileId = new DeviceProfileId(UUID.randomUUID()); + + @Before + public void setUp() { + } + + @Test + public void testFindTenantDevices() { + DeviceProfile deviceProfile = new DeviceProfile(deviceProfileId); + deviceProfile.setName("thermostat"); + deviceProfile.setDefault(false); + deviceProfile.setType(DeviceProfileType.DEFAULT); + addOrUpdate(EntityType.DEVICE_PROFILE, deviceProfile); + + UUID asset1 = createAsset("A1"); + UUID asset2 = createAsset("A2"); + UUID device1 = createDevice(null, deviceProfileId.getId(), "D1"); + UUID device2 = createDevice(null, deviceProfileId.getId(), "D2"); + + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + createRelation(EntityType.ASSET, asset2, EntityType.DEVICE, device2, "Contains"); + + // find all devices of asset A1 + PageData relationsResult = findData(null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("thermostat")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + Assert.assertTrue(checkContains(relationsResult, device2)); + + // find all devices with max level = 1 + relationsResult = findData(null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("thermostat")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + + // find all devices with asset type = default + relationsResult = findData(null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("default")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all devices last level only, level = 2 + relationsResult = findData(null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, true, Arrays.asList("thermostat")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device2)); + Assert.assertTrue(checkContains(relationsResult, device1)); + + // find all devices last level only, level = 1 + relationsResult = findData(null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 1, true, Arrays.asList("thermostat")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + } + + @Test + public void testFindCustomerDevices() { + DeviceProfile deviceProfile = new DeviceProfile(deviceProfileId); + deviceProfile.setName("thermostat"); + deviceProfile.setDefault(false); + deviceProfile.setType(DeviceProfileType.DEFAULT); + addOrUpdate(EntityType.DEVICE_PROFILE, deviceProfile); + + UUID asset1 = createAsset(customerId.getId(), defaultAssetProfileId, "A1"); + UUID asset2 = createAsset(customerId.getId(), defaultAssetProfileId, "A2"); + UUID device1 = createDevice(customerId.getId(), deviceProfileId.getId(), "D1"); + UUID device2 = createDevice(customerId.getId(), defaultDeviceProfileId, "D2"); + + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset2, EntityType.DEVICE, device2, "Contains"); + + // find all devices of type "thermostat" + PageData relationsResult = findData(customerId, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("thermostat")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + + // find all assets of root asset with profile "Office" and "default" + relationsResult = findData(customerId, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("thermostat", "default")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + Assert.assertTrue(checkContains(relationsResult, device2)); + + // find all assets with other customer + relationsResult = findData(new CustomerId(UUID.randomUUID()), new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("thermostat")); + Assert.assertEquals(0, relationsResult.getData().size()); + } + + private PageData findData(CustomerId customerId, EntityId rootId, + EntitySearchDirection direction, String relationType, int maxLevel, boolean lastLevelOnly, List deviceTypes) { + DeviceSearchQueryFilter filter = new DeviceSearchQueryFilter(); + filter.setRootEntity(rootId); + filter.setDirection(direction); + filter.setRelationType(relationType); + filter.setDeviceTypes(deviceTypes); + filter.setFetchLastLevelOnly(lastLevelOnly); + filter.setMaxLevel(maxLevel); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "D"); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + return repository.findEntityDataByQuery(tenantId, customerId, query, false); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceTypeFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceTypeFilterTest.java new file mode 100644 index 0000000000..e07dfc98e4 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceTypeFilterTest.java @@ -0,0 +1,141 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.DeviceTypeFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; + +import java.util.Arrays; +import java.util.Collections; +import java.util.UUID; + +public class DeviceTypeFilterTest extends AbstractEDQTest { + + private final DeviceProfileId loraProfileId = new DeviceProfileId(UUID.randomUUID()); + + @Before + public void setUp() { + DeviceProfile deviceProfile = new DeviceProfile(loraProfileId); + deviceProfile.setName("LoRa"); + deviceProfile.setDefault(false); + deviceProfile.setType(DeviceProfileType.DEFAULT); + addOrUpdate(EntityType.DEVICE_PROFILE, deviceProfile); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setDeviceProfileId(loraProfileId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + addOrUpdate(EntityType.DEVICE, device); + + var result = repository.findEntityDataByQuery(tenantId, null, getDeviceTypeQuery("LoRa"), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + + result = repository.findEntityDataByQuery(tenantId, null, getDeviceTypeQuery("Not LoRa"), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, getDeviceTypeQuery("LoRa"), false); + Assert.assertEquals(1, result.getTotalElements()); + result = repository.findEntityDataByQuery(tenantId, customerId, getDeviceTypeQuery("default"), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + device.setDeviceProfileId(loraProfileId); + + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(new LatestTsKv(deviceId, new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, getDeviceTypeQuery("LoRa"), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, getDeviceTypeQuery("LoRa"), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + private static EntityDataQuery getDeviceTypeQuery(String deviceType) { + DeviceTypeFilter filter = new DeviceTypeFilter(); + filter.setDeviceTypes(Collections.singletonList(deviceType)); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "state"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); + predicate.setValue(new FilterPredicateValue<>("LoRa-")); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, Arrays.asList(nameFilter)); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeSearchQueryFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeSearchQueryFilterTest.java new file mode 100644 index 0000000000..f0911570ea --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeSearchQueryFilterTest.java @@ -0,0 +1,116 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EdgeSearchQueryFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +public class EdgeSearchQueryFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @Test + public void testFindDevicesManagesByTenant() { + UUID edge1 = createEdge("E1"); + UUID edge2 = createEdge("E2"); + UUID device1 = createDevice("D1"); + UUID device2 = createDevice("D2"); + UUID device3 = createDevice("D3"); + + createRelation(EntityType.EDGE, edge1, EntityType.DEVICE, device1, "Manages"); + createRelation(EntityType.EDGE, edge2, EntityType.DEVICE, device2, "Manages"); + createRelation(EntityType.EDGE, edge2, EntityType.DEVICE, device3, "Manages"); + + // find devices managed by edge + PageData relationsResult = findData(null, new DeviceId(device1), + EntitySearchDirection.TO, "Manages", 2, false, Arrays.asList("default")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, edge1)); + + // find devices managed by edge with non-existing type + relationsResult = findData(null, new DeviceId(device1), + EntitySearchDirection.TO, "Manages", 1, false, Arrays.asList("non-existing type")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all entity views last level only, level = 2 + relationsResult = findData(null, new DeviceId(device1), + EntitySearchDirection.TO, "Manages", 2, true, Arrays.asList("default")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, edge1)); + } + + @Test + public void testFindCustomerEdges() { + UUID edge1 = createEdge(customerId, "E1"); + UUID edge2 = createEdge(customerId, "E2"); + createRelation(EntityType.CUSTOMER, customerId.getId(), EntityType.EDGE, edge1, "Manages"); + createRelation(EntityType.CUSTOMER, customerId.getId(), EntityType.EDGE, edge2, "Manages"); + + // find all edges managed by customer + PageData relationsResult = findData(customerId, customerId, + EntitySearchDirection.FROM, "Manages", 2, false, Arrays.asList("default")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, edge1)); + Assert.assertTrue(checkContains(relationsResult, edge2)); + + // find all edges managed by customer with non-existing type + relationsResult = findData(customerId, customerId, + EntitySearchDirection.FROM, "Manages", 2, false, Arrays.asList("non existing")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all entity views with other customer + relationsResult = findData(new CustomerId(UUID.randomUUID()), customerId, + EntitySearchDirection.FROM, "Manages", 2, false, Arrays.asList("default")); + Assert.assertEquals(0, relationsResult.getData().size()); + } + + private PageData findData(CustomerId customerId, EntityId rootId, + EntitySearchDirection direction, String relationType, int maxLevel, boolean lastLevelOnly, List edgeTypes) { + EdgeSearchQueryFilter filter = new EdgeSearchQueryFilter(); + filter.setRootEntity(rootId); + filter.setDirection(direction); + filter.setRelationType(relationType); + filter.setEdgeTypes(edgeTypes); + filter.setFetchLastLevelOnly(lastLevelOnly); + filter.setMaxLevel(maxLevel); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "E"); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + return repository.findEntityDataByQuery(tenantId, customerId, query, false); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeTypeFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeTypeFilterTest.java new file mode 100644 index 0000000000..1d7bbef011 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeTypeFilterTest.java @@ -0,0 +1,176 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.EdgeTypeFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +public class EdgeTypeFilterTest extends AbstractEDQTest { + + private Edge edge; + private Edge edge2; + private Edge edge3; + + + @Before + public void setUp() { + edge = buildEdge("default", "Edge 1"); + edge2 = buildEdge("default", "Edge 2"); + edge3 = buildEdge("edge v2", "Edge 3"); + addOrUpdate(EntityType.EDGE, edge); + addOrUpdate(EntityType.EDGE, edge2); + addOrUpdate(EntityType.EDGE, edge3); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantEdges() { + // find edges with type "default" + var result = repository.findEntityDataByQuery(tenantId, null, getEdgeTypeQuery(Collections.singletonList("default"), null, null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Optional firstView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("Edge 1")).findFirst(); + assertThat(firstView).isPresent(); + assertThat(firstView.get().getEntityId()).isEqualTo(edge.getId()); + assertThat(firstView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(edge.getCreatedTime())); + + // find edges with types "default" and "edge v2" + result = repository.findEntityDataByQuery(tenantId, null, getEdgeTypeQuery(Arrays.asList("default", "edge v2"), null, null), false); + + Assert.assertEquals(3, result.getTotalElements()); + Optional thirdView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("Edge 3")).findFirst(); + assertThat(thirdView).isPresent(); + assertThat(thirdView.get().getEntityId()).isEqualTo(edge3.getId()); + assertThat(thirdView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(edge.getCreatedTime())); + + // find entity view with type "day 3" + result = repository.findEntityDataByQuery(tenantId, null, getEdgeTypeQuery(List.of("edge v3"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find entity view with name "%Edge%" + result = repository.findEntityDataByQuery(tenantId, null, getEdgeTypeQuery(List.of("default"), "%Edge%", null), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find entity view with name "Edge 1" + result = repository.findEntityDataByQuery(tenantId, null, getEdgeTypeQuery(List.of("default"), "Edge 1", null), false); + Assert.assertEquals(1, result.getTotalElements()); + + // find entity view with name "%Edge 4%" + result = repository.findEntityDataByQuery(tenantId, null, getEdgeTypeQuery(List.of("default"), "%Edge 4%", null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find entity view with key filter: name contains "Edge" + KeyFilter containsNameFilter = getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "Edge", true); + result = repository.findEntityDataByQuery(tenantId, null, getEdgeTypeQuery(List.of("default"), null, List.of(containsNameFilter)), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find entity view with key filter: name starts with "edge" and matches case + KeyFilter startsWithNameFilter = getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation.STARTS_WITH, "edge", false); + result = repository.findEntityDataByQuery(tenantId, null, getEdgeTypeQuery(List.of("default"), null, List.of(startsWithNameFilter)), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerEdges() { + addOrUpdate(new LatestTsKv(edge.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, getEdgeTypeQuery(List.of("default"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + edge.setCustomerId(customerId); + edge2.setCustomerId(customerId); + edge3.setCustomerId(customerId); + addOrUpdate(EntityType.EDGE, edge); + addOrUpdate(EntityType.EDGE, edge2); + addOrUpdate(EntityType.EDGE, edge3); + + result = repository.findEntityDataByQuery(tenantId, customerId, getEdgeTypeQuery(List.of("default"), null, null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Optional firstView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("Edge 1")).findFirst(); + assertThat(firstView).isPresent(); + assertThat(firstView.get().getEntityId()).isEqualTo(edge.getId()); + assertThat(firstView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(edge.getCreatedTime())); + + result = repository.findEntityDataByQuery(tenantId, customerId, getEdgeTypeQuery(List.of("edge v3"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + private Edge buildEdge(String type, String name) { + Edge edge = new Edge(); + edge.setId(new EdgeId(UUID.randomUUID())); + edge.setTenantId(tenantId); + edge.setType(type); + edge.setName(name); + edge.setCreatedTime(42L); + return edge; + } + + private static EntityDataQuery getEdgeTypeQuery(List edgeTypes, String edgeNameFilter, List keyFilters) { + EdgeTypeFilter filter = new EdgeTypeFilter(); + filter.setEdgeTypes(edgeTypes); + filter.setEdgeNameFilter(edgeNameFilter); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + } + + private static KeyFilter getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation operation, String predicateValue, boolean ignoreCase) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(ignoreCase); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(predicateValue)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityListFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityListFilterTest.java new file mode 100644 index 0000000000..dc48bd5438 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityListFilterTest.java @@ -0,0 +1,143 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; + +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +public class EntityListFilterTest extends AbstractEDQTest { + + private Device device; + private Device device2; + private Device device3; + + + @Before + public void setUp() { + device = buildDevice("LoRa-1"); + device2 = buildDevice("LoRa-2"); + device3 = buildDevice("Parking-Sensor-1"); + + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(EntityType.DEVICE, device2); + addOrUpdate(EntityType.DEVICE, device3); + + addOrUpdate(new LatestTsKv(device.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "enabled")), 0L)); + addOrUpdate(new LatestTsKv(device2.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "disabled")), 0L)); + addOrUpdate(new LatestTsKv(device3.getId(), new BasicTsKvEntry(43, new BooleanDataEntry("free", true)), 0L)); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantDevice() { + // get entity list + var result = repository.findEntityDataByQuery(tenantId, null, getEntityListDataQuery(EntityType.DEVICE, List.of(device.getId().getId().toString())), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(device.getId(), first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + Assert.assertEquals("enabled", first.getLatest().get(EntityKeyType.TIME_SERIES).get("state").getValue()); + + result = repository.findEntityDataByQuery(tenantId, null, getEntityListDataQuery(EntityType.DEVICE,List.of(device.getId().getId().toString(), device2.getId().getId().toString())), false); + Assert.assertEquals(2, result.getTotalElements()); + + result = repository.findEntityDataByQuery(tenantId, null, getEntityListDataQuery(EntityType.DEVICE, List.of(UUID.randomUUID().toString())), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerDevice() { + var result = repository.findEntityDataByQuery(tenantId, customerId, getEntityListDataQuery(EntityType.DEVICE, List.of(device.getId().getId().toString())), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, getEntityListDataQuery(EntityType.DEVICE, List.of(device.getId().getId().toString())), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(device.getId(), first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + Assert.assertEquals("enabled", first.getLatest().get(EntityKeyType.TIME_SERIES).get("state").getValue()); + } + + private Device buildDevice(String name) { + Device device = new Device(); + device.setId(new DeviceId(UUID.randomUUID())); + device.setTenantId(tenantId); + device.setName(name); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + return device; + } + + private static EntityDataQuery getEntityListDataQuery(EntityType entityType, List ids) { + EntityListFilter filter = new EntityListFilter(); + filter.setEntityType(entityType); + filter.setEntityList(ids); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "state"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + KeyFilter nameFilter = getNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "LoRa-"); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, Arrays.asList(nameFilter)); + } + + private static KeyFilter getNameKeyFilter(StringFilterPredicate.StringOperation operation, String value) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(value)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityNameFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityNameFilterTest.java new file mode 100644 index 0000000000..3e47c245ef --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityNameFilterTest.java @@ -0,0 +1,131 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.EntityNameFilter; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; + +import java.util.Arrays; +import java.util.UUID; + +public class EntityNameFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + addOrUpdate(EntityType.DEVICE, device); + + var result = repository.findEntityDataByQuery(tenantId, null, getDeviceNameQuery("LoRa"), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + + result = repository.findEntityDataByQuery(tenantId, null, getDeviceNameQuery("Not LoRa"), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, null, getDeviceNameQuery("%1"), false); + Assert.assertEquals(1, result.getTotalElements()); + result = repository.findEntityDataByQuery(tenantId, null, getDeviceNameQuery("L%"), false); + Assert.assertEquals(1, result.getTotalElements()); + } + + @Test + public void testFindCustomerDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(new LatestTsKv(deviceId, new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, getDeviceNameQuery("LoRa"), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, getDeviceNameQuery("LoRa"), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + private static EntityDataQuery getDeviceNameQuery(String entityNameFilter) { + EntityNameFilter filter = new EntityNameFilter(); + filter.setEntityType(EntityType.DEVICE); + filter.setEntityNameFilter(entityNameFilter); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "state"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); + predicate.setValue(new FilterPredicateValue<>("LoRa-")); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, Arrays.asList(nameFilter)); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityTypeFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityTypeFilterTest.java new file mode 100644 index 0000000000..8fab142dc7 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityTypeFilterTest.java @@ -0,0 +1,146 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; + +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class EntityTypeFilterTest extends AbstractEDQTest { + + private Device device; + private Device device2; + private Device device3; + + @Before + public void setUp() { + device = buildDevice("LoRa-1"); + device2 = buildDevice("LoRa-2"); + device3 = buildDevice("Parking-Sensor-1"); + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(EntityType.DEVICE, device2); + addOrUpdate(EntityType.DEVICE, device3); + addOrUpdate(new LatestTsKv(device.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "enabled")), 0L)); + addOrUpdate(new LatestTsKv(device2.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "disabled")), 0L)); + addOrUpdate(new LatestTsKv(device3.getId(), new BasicTsKvEntry(43, new BooleanDataEntry("free", true)), 0L)); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantDeviceEntities() { + // find all tenant devices + var result = repository.findEntityDataByQuery(tenantId, null, getEntityTypeQuery(EntityType.DEVICE, null), false); + + Assert.assertEquals(3, result.getTotalElements()); + var first = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("LoRa-1")).findAny(); + assertThat(first).isPresent(); + assertThat(first.get().getEntityId()).isEqualTo(device.getId()); + assertThat(first.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(device.getCreatedTime())); + assertThat(first.get().getLatest().get(EntityKeyType.TIME_SERIES).get("state").getValue()).isEqualTo("enabled"); + + // find all tenant devices with filter by name + KeyFilter keyFilter = getDeviceNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "Lora", true); + result = repository.findEntityDataByQuery(tenantId, null, getEntityTypeQuery(EntityType.DEVICE, List.of(keyFilter)), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find asset entities + result = repository.findEntityDataByQuery(tenantId, null, getEntityTypeQuery(EntityType.ASSET, null), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerDeviceEntities() { + var result = repository.findEntityDataByQuery(tenantId, customerId, getEntityTypeQuery(EntityType.DEVICE, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, getEntityTypeQuery(EntityType.DEVICE, null), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(device.getId(), first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + Assert.assertEquals("enabled", first.getLatest().get(EntityKeyType.TIME_SERIES).get("state").getValue()); + + result = repository.findEntityDataByQuery(tenantId, customerId, getEntityTypeQuery(EntityType.ASSET, null), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + private Device buildDevice(String name) { + Device device = new Device(); + device.setId(new DeviceId(UUID.randomUUID())); + device.setTenantId(tenantId); + device.setName(name); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + return device; + } + + private static EntityDataQuery getEntityTypeQuery(EntityType entityType, List keyFilters) { + EntityTypeFilter filter = new EntityTypeFilter(); + filter.setEntityType(entityType); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + } + + private static KeyFilter getDeviceNameKeyFilter(StringFilterPredicate.StringOperation operation, String predicateValue, boolean ignoreCase) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(ignoreCase); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(predicateValue)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewSearchQueryFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewSearchQueryFilterTest.java new file mode 100644 index 0000000000..fb32759045 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewSearchQueryFilterTest.java @@ -0,0 +1,130 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityViewSearchQueryFilter; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +public class EntityViewSearchQueryFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @Test + public void testFindTenantEntityViews() { + UUID asset1 = createAsset("A1"); + UUID device1 = createDevice("D1"); + UUID device2 = createDevice("D2"); + UUID deviceView1 = createView("V1"); + UUID deviceView2 = createView("V2"); + + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device2, "Contains"); + createRelation(EntityType.DEVICE, device1, EntityType.ENTITY_VIEW, deviceView1, "Contains"); + createRelation(EntityType.DEVICE, device2, EntityType.ENTITY_VIEW, deviceView2, "Contains"); + + // find all entity views of asset A1 + PageData relationsResult = findData(null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("default")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, deviceView1)); + Assert.assertTrue(checkContains(relationsResult, deviceView2)); + + // find all entity views with max level = 1 + relationsResult = findData(null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("default")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all entity views with type "day 1" + relationsResult = findData(null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("day 1")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all entity views last level only, level = 2 + relationsResult = findData(null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, true, Arrays.asList("default")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, deviceView1)); + Assert.assertTrue(checkContains(relationsResult, deviceView2)); + } + + @Test + public void testFindCustomerDevices() { + UUID asset1 = createAsset(customerId.getId(), defaultAssetProfileId, "A1"); + UUID device1 = createDevice(customerId.getId(), defaultDeviceProfileId, "D1"); + UUID device2 = createDevice(customerId.getId(), defaultDeviceProfileId, "D2"); + UUID deviceView1 = createView(customerId.getId(), "day 1", "V1"); + UUID deviceView2 = createView(customerId.getId(), "day 1", "V2"); + + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device2, "Contains"); + createRelation(EntityType.DEVICE, device1, EntityType.ENTITY_VIEW, deviceView1, "Contains"); + createRelation(EntityType.DEVICE, device2, EntityType.ENTITY_VIEW, deviceView2, "Contains"); + + // find all entity views of type "day 1" + PageData relationsResult = findData(customerId, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("day 1")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, deviceView1)); + Assert.assertTrue(checkContains(relationsResult, deviceView2)); + + // find all entity views of type "day 2" + relationsResult = findData(customerId, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("day 2")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all entity views with other customer + relationsResult = findData(new CustomerId(UUID.randomUUID()), new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("thermostat")); + Assert.assertEquals(0, relationsResult.getData().size()); + } + + private PageData findData(CustomerId customerId, EntityId rootId, + EntitySearchDirection direction, String relationType, int maxLevel, boolean lastLevelOnly, List entityViewTypes) { + EntityViewSearchQueryFilter filter = new EntityViewSearchQueryFilter(); + filter.setRootEntity(rootId); + filter.setDirection(direction); + filter.setRelationType(relationType); + filter.setEntityViewTypes(entityViewTypes); + filter.setFetchLastLevelOnly(lastLevelOnly); + filter.setMaxLevel(maxLevel); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "V"); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + return repository.findEntityDataByQuery(tenantId, customerId, query, false); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewTypeFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewTypeFilterTest.java new file mode 100644 index 0000000000..bf1d329127 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewTypeFilterTest.java @@ -0,0 +1,176 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.EntityViewTypeFilter; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +public class EntityViewTypeFilterTest extends AbstractEDQTest { + + private EntityView entityView; + private EntityView entityView2; + private EntityView entityView3; + + + @Before + public void setUp() { + entityView = buildEntityView("day 1", "day 1 lora 1 view"); + entityView2 = buildEntityView("day 1", "day 1 lora 2 view"); + entityView3 = buildEntityView("day 2", "day 2 lora 1 view"); + addOrUpdate(EntityType.ENTITY_VIEW, entityView); + addOrUpdate(EntityType.ENTITY_VIEW, entityView2); + addOrUpdate(EntityType.ENTITY_VIEW, entityView3); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantEntityView() { + // find entity view with type "day 1" + var result = repository.findEntityDataByQuery(tenantId, null, getEntityViewTypeQuery(Collections.singletonList("day 1"), null, null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Optional firstView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("day 1 lora 1 view")).findFirst(); + assertThat(firstView).isPresent(); + assertThat(firstView.get().getEntityId()).isEqualTo(entityView.getId()); + assertThat(firstView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(entityView.getCreatedTime())); + + // find entity view with types "day 1" and "day 2" + result = repository.findEntityDataByQuery(tenantId, null, getEntityViewTypeQuery(Arrays.asList("day 1", "day 2"), null, null), false); + + Assert.assertEquals(3, result.getTotalElements()); + Optional thirdView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("day 2 lora 1 view")).findFirst(); + assertThat(thirdView).isPresent(); + assertThat(thirdView.get().getEntityId()).isEqualTo(entityView3.getId()); + assertThat(thirdView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(entityView.getCreatedTime())); + + // find entity view with type "day 3" + result = repository.findEntityDataByQuery(tenantId, null, getEntityViewTypeQuery(Collections.singletonList("day 3"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find entity view with name "%Lora%" + result = repository.findEntityDataByQuery(tenantId, null, getEntityViewTypeQuery(Collections.singletonList("day 1"), "%day 1 lora%", null), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find entity view with name "Lora 1 device view" + result = repository.findEntityDataByQuery(tenantId, null, getEntityViewTypeQuery(Collections.singletonList("day 1"), "day 1 lora 1 view", null), false); + Assert.assertEquals(1, result.getTotalElements()); + + // find entity view with name "%Parking sensor%" + result = repository.findEntityDataByQuery(tenantId, null, getEntityViewTypeQuery(Collections.singletonList("day 1"), "%day 3 lora%", null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find entity view with key filter: name contains "Lora" + KeyFilter containsNameFilter = getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "Lora", true); + result = repository.findEntityDataByQuery(tenantId, null, getEntityViewTypeQuery(Collections.singletonList("day 1"), null, Arrays.asList(containsNameFilter)), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find entity view with key filter: name starts with "lora" and matches case + KeyFilter startsWithNameFilter = getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation.STARTS_WITH, "lora", false); + result = repository.findEntityDataByQuery(tenantId, null, getEntityViewTypeQuery(Collections.singletonList("day 1"), null, Arrays.asList(startsWithNameFilter)), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerEntityView() { + addOrUpdate(new LatestTsKv(entityView.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, getEntityViewTypeQuery(Collections.singletonList("day 1"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + entityView.setCustomerId(customerId); + entityView2.setCustomerId(customerId); + entityView3.setCustomerId(customerId); + addOrUpdate(EntityType.ENTITY_VIEW, entityView); + addOrUpdate(EntityType.ENTITY_VIEW, entityView2); + addOrUpdate(EntityType.ENTITY_VIEW, entityView3); + + result = repository.findEntityDataByQuery(tenantId, customerId, getEntityViewTypeQuery(Collections.singletonList("day 1"), null, null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Optional firstView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("day 1 lora 1 view")).findFirst(); + assertThat(firstView).isPresent(); + assertThat(firstView.get().getEntityId()).isEqualTo(entityView.getId()); + assertThat(firstView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(entityView.getCreatedTime())); + + result = repository.findEntityDataByQuery(tenantId, customerId, getEntityViewTypeQuery(Collections.singletonList("day 3"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + private EntityView buildEntityView(String type, String name) { + EntityView entityView = new EntityView(); + entityView.setId(new EntityViewId(UUID.randomUUID())); + entityView.setTenantId(tenantId); + entityView.setType(type); + entityView.setName(name); + entityView.setCreatedTime(42L); + return entityView; + } + + private static EntityDataQuery getEntityViewTypeQuery(List assetTypes, String entityViewNameFilter, List keyFilters) { + EntityViewTypeFilter filter = new EntityViewTypeFilter(); + filter.setEntityViewTypes(assetTypes); + filter.setEntityViewNameFilter(entityViewNameFilter); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + } + + private static KeyFilter getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation operation, String predicateValue, boolean ignoreCase) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(ignoreCase); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(predicateValue)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RelationsQueryFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RelationsQueryFilterTest.java new file mode 100644 index 0000000000..094d46f977 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RelationsQueryFilterTest.java @@ -0,0 +1,160 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +public class RelationsQueryFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @Test + public void testFindTenantDevices() { + UUID ta1 = createAsset("T A1"); + UUID ta2 = createAsset("T A2"); + UUID da1 = createDevice("T D1"); + UUID da2 = createDevice(customerId, "T D2"); + UUID da3 = createDevice("NOT MATCHING D3"); + + // A1 --Contains--> A2, A1 --Contains--> D1. A1 --Manages--> D2. + createRelation(EntityType.ASSET, ta1, EntityType.ASSET, ta2, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da1, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da2, "Manages"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da3, "Contains"); + + PageData relationsResult = filter(new AssetId(ta1), new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, ta2)); + Assert.assertTrue(checkContains(relationsResult, da1)); + + relationsResult = filter(new AssetId(ta1), new RelationEntityTypeFilter("Manages", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, da2)); + } + + @Test + public void testFindTenantDevicesLastLevelOnly() { + UUID root = createAsset("T ROOT"); + + UUID ta1 = createAsset("T A1 NO MORE RELATIONS"); + UUID ta2 = createAsset("T A2"); + UUID da1 = createDevice("T D1"); + UUID da2 = createDevice(customerId, "T D2"); + UUID da3 = createDevice(customerId, "T D3"); + UUID da4 = createDevice(customerId, "T D4"); // Lvl 4 + + // ROOT --Contains--> A1, A2; A2 --Contains--> D1, D2; D2 --Contains--> D3. + createRelation(EntityType.ASSET, root, EntityType.ASSET, ta1, "Contains"); + createRelation(EntityType.ASSET, root, EntityType.ASSET, ta2, "Contains"); + createRelation(EntityType.ASSET, ta2, EntityType.DEVICE, da1, "Contains"); + createRelation(EntityType.ASSET, ta2, EntityType.DEVICE, da2, "Contains"); + createRelation(EntityType.ASSET, da2, EntityType.DEVICE, da3, "Contains"); + createRelation(EntityType.ASSET, da3, EntityType.DEVICE, da4, "Contains"); + + PageData relationsResult = filter(null, new AssetId(root), 1, true, + new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, ta1)); + Assert.assertTrue(checkContains(relationsResult, ta2)); + + relationsResult = filter(null, new AssetId(root), 2, true, + new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(3, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, ta1)); + Assert.assertTrue(checkContains(relationsResult, da1)); + Assert.assertTrue(checkContains(relationsResult, da2)); + + relationsResult = filter(null, new AssetId(root), 3, true, + new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(3, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, ta1)); + Assert.assertTrue(checkContains(relationsResult, da1)); + Assert.assertTrue(checkContains(relationsResult, da3)); + + relationsResult = filter(null, new AssetId(root), 4, true, + new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(3, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, ta1)); + Assert.assertTrue(checkContains(relationsResult, da1)); + Assert.assertTrue(checkContains(relationsResult, da4)); + + } + + @Test + public void testFindCustomerDevices() { + UUID ta1 = createAsset("T A1"); + UUID ta2 = createAsset("T A2"); + UUID da1 = createDevice(customerId, "T D1"); + UUID da2 = createDevice("T D2"); + + // A1 --Contains--> A2, A1 --Contains--> D1. A1 --Manages--> D2. + createRelation(EntityType.ASSET, ta1, EntityType.ASSET, ta2, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da1, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da2, "Manages"); + + PageData relationsResult = filter(customerId, new AssetId(ta1), new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, da1)); + + relationsResult = filter(customerId, new AssetId(ta1), new RelationEntityTypeFilter("Manages", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(0, relationsResult.getData().size()); + } + + private PageData filter(EntityId rootId, RelationEntityTypeFilter... relationEntityTypeFilters) { + return filter(null, rootId, relationEntityTypeFilters); + } + + private PageData filter(CustomerId customerId, EntityId rootId, RelationEntityTypeFilter... relationEntityTypeFilters) { + return filter(customerId, rootId, 3, false, relationEntityTypeFilters); + } + + private PageData filter(CustomerId customerId, EntityId rootId, int maxLevel, boolean lastLevelOnly, RelationEntityTypeFilter... relationEntityTypeFilters) { + RelationsQueryFilter filter = new RelationsQueryFilter(); + filter.setRootEntity(rootId); + filter.setFilters(Arrays.asList(relationEntityTypeFilters)); + filter.setDirection(EntitySearchDirection.FROM); + filter.setFetchLastLevelOnly(lastLevelOnly); + filter.setMaxLevel(maxLevel); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "T"); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + return repository.findEntityDataByQuery(tenantId, customerId, query, false); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java new file mode 100644 index 0000000000..6c7444c92a --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java @@ -0,0 +1,434 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.fields.DeviceFields; +import org.thingsboard.server.common.data.edqs.fields.DeviceProfileFields; +import org.thingsboard.server.common.data.query.BooleanFilterPredicate; +import org.thingsboard.server.common.data.query.BooleanFilterPredicate.BooleanOperation; +import org.thingsboard.server.common.data.query.ComplexFilterPredicate; +import org.thingsboard.server.common.data.query.ComplexFilterPredicate.ComplexOperation; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.NumericFilterPredicate; +import org.thingsboard.server.common.data.query.NumericFilterPredicate.NumericOperation; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.query.StringFilterPredicate.StringOperation; +import org.thingsboard.server.edqs.data.DeviceData; +import org.thingsboard.server.edqs.data.EntityProfileData; +import org.thingsboard.server.edqs.data.dp.BoolDataPoint; +import org.thingsboard.server.edqs.data.dp.DoubleDataPoint; +import org.thingsboard.server.edqs.data.dp.LongDataPoint; +import org.thingsboard.server.edqs.data.dp.StringDataPoint; +import org.thingsboard.server.edqs.query.DataKey; +import org.thingsboard.server.edqs.query.EdqsFilter; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.List; +import java.util.UUID; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + + +public class RepositoryUtilsTest { + + private static Stream deviceNameFilters() { + return Stream.of(Arguments.of(null, getNameFilter(StringOperation.STARTS_WITH, "lora"), true), + Arguments.of("loranet device 123", getNameFilter(StringOperation.STARTS_WITH, "lora"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "ra"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "123"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "device"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.EQUAL, "loranet 123"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.EQUAL, "loranet "), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_EQUAL, "loranet"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_EQUAL, "loranet 123"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loranet"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loranet123"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_CONTAINS, "loranet123"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_CONTAINS, "loranet"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.IN, "loranet 123, loranet 124"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.IN, "loranet 125, loranet 126"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_IN, "loranet 125, loranet 126"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_IN, "loranet 123, loranet 126"), false) + ); + } + + @ParameterizedTest + @MethodSource("deviceNameFilters") + public void testFilterByDeviceName(String deviceName, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(deviceName).build()); + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream createdTimeFilters() { + return Stream.of(Arguments.of(1000, getCreatedTimeFilter(NumericOperation.EQUAL, 1000), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.EQUAL, 1001), false), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.NOT_EQUAL, 1000), false), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.NOT_EQUAL, 1001), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.GREATER, 999), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.GREATER, 1000), false), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.GREATER_OR_EQUAL, 1000), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.GREATER_OR_EQUAL, 1001), false), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.LESS, 1001), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.LESS, 1000), false), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.LESS_OR_EQUAL, 1000), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.LESS_OR_EQUAL, 999), false) + ); + } + + @ParameterizedTest + @MethodSource("createdTimeFilters") + public void testFilterDevicesByCreatedTime(long createdTime, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().createdTime(createdTime).build()); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream deviceNameAndTypeFilter() { + return Stream.of( + Arguments.of("loranet 123", "thermostat", List.of(getNameFilter(StringOperation.STARTS_WITH, "lo"), getTypeFilter(StringOperation.EQUAL, "thermostat")), true), + Arguments.of("loranet 123", "thermostat", List.of(getNameFilter(StringOperation.STARTS_WITH, "net"), getTypeFilter(StringOperation.EQUAL, "thermostat")), false), + Arguments.of("loranet 123", "thermostat", List.of(getNameFilter(StringOperation.STARTS_WITH, "lo"), getTypeFilter(StringOperation.EQUAL, "sensor1")), false), + Arguments.of("loranet 123", "thermostat", List.of(getNameFilter(StringOperation.STARTS_WITH, "net"), getTypeFilter(StringOperation.EQUAL, "sensor1")), false)); + } + + @ParameterizedTest + @MethodSource("deviceNameAndTypeFilter") + public void testFilterByDeviceNameAndDeviceType(String deviceName, String deviceType, List keyFilters, boolean result) { + UUID deviceProfileId = UUID.randomUUID(); + EntityProfileData deviceProfile = new EntityProfileData(deviceProfileId, EntityType.DEVICE_PROFILE); + deviceProfile.setFields(DeviceProfileFields.builder().name(deviceType).build()); + + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(deviceName).deviceProfileId(deviceProfileId).type(deviceType).build()); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, keyFilters)).isEqualTo(result); + } + + private static Stream deviceNameComplexFilters() { + return Stream.of(Arguments.of(null, List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.AND, StringOperation.ENDS_WITH, "123")), true), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.AND, StringOperation.ENDS_WITH, "123")), true), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.AND, StringOperation.ENDS_WITH, "124")), false), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.OR, StringOperation.STARTS_WITH, "net")), true), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "net", ComplexOperation.OR, StringOperation.STARTS_WITH, "the")), false), + Arguments.of("loranet123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.OR, StringOperation.STARTS_WITH, "the", + ComplexOperation.AND, StringOperation.ENDS_WITH, "123")), true), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "net", ComplexOperation.OR, StringOperation.STARTS_WITH, "the", + ComplexOperation.OR, StringOperation.ENDS_WITH, "123")), true), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "net", ComplexOperation.OR, StringOperation.STARTS_WITH, "the", + ComplexOperation.AND, StringOperation.ENDS_WITH, "123")), false), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.OR, StringOperation.STARTS_WITH, "the", + ComplexOperation.AND, StringOperation.ENDS_WITH, "124")), false) + ); + } + + @ParameterizedTest + @MethodSource("deviceNameComplexFilters") + public void testFilterByDeviceNameComplexFilters(String deviceName, List keyFilters, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(deviceName).build()); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, keyFilters)).isEqualTo(result); + } + + private static Stream deviceTemperatureFilters() { + return Stream.of(Arguments.of(22.8, getTemperatureFilter(NumericOperation.EQUAL, 22.8), true), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.EQUAL, 22.9), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.NOT_EQUAL, 22.8), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.NOT_EQUAL, 22.9), true), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.GREATER, 22.0), true), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.GREATER, 23.0), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.GREATER_OR_EQUAL, 22.8), true), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.GREATER_OR_EQUAL, 23.0), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.LESS, 23.0), true), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.LESS, 22.0), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.LESS_OR_EQUAL, 22.0), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.LESS_OR_EQUAL, 22.8), true) + ); + } + + @ParameterizedTest + @MethodSource("deviceTemperatureFilters") + public void testFilterByDeviceTemperature(double tempValue, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putTs(5, new DoubleDataPoint(System.currentTimeMillis(), tempValue)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream deviceTemperatureComplexFilters() { + return Stream.of(Arguments.of(22.8, getComplexTemperatureFilter(NumericOperation.GREATER_OR_EQUAL, 22.8, ComplexOperation.AND, NumericOperation.LESS_OR_EQUAL, 30), true), + Arguments.of(22.8, getComplexTemperatureFilter(NumericOperation.GREATER, 23.5, ComplexOperation.AND, NumericOperation.LESS_OR_EQUAL, 30), false), + Arguments.of(22.8, getComplexComplexTemperatureFilter(NumericOperation.GREATER, 22.0, ComplexOperation.AND, NumericOperation.LESS_OR_EQUAL, 30, ComplexOperation.OR, NumericOperation.GREATER, 35), true), + Arguments.of(22.8, getComplexComplexTemperatureFilter(NumericOperation.GREATER, 22.0, ComplexOperation.AND, NumericOperation.LESS_OR_EQUAL, 30, ComplexOperation.AND, NumericOperation.EQUAL, 22.8), true) + ); + } + + @ParameterizedTest + @MethodSource("deviceTemperatureComplexFilters") + public void testComplexFilterByDeviceTemperature(double tempValue, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putTs(5, new DoubleDataPoint(System.currentTimeMillis(), tempValue)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream deviceHumidityFilters() { + return Stream.of(Arguments.of(60, getHumidityFilter(NumericOperation.EQUAL, 60), true), + Arguments.of(60, getHumidityFilter(NumericOperation.EQUAL, 61), false), + Arguments.of(60, getHumidityFilter(NumericOperation.NOT_EQUAL, 60), false), + Arguments.of(60, getHumidityFilter(NumericOperation.NOT_EQUAL, 61), true), + Arguments.of(60, getHumidityFilter(NumericOperation.GREATER, 59), true), + Arguments.of(60, getHumidityFilter(NumericOperation.GREATER, 60), false), + Arguments.of(60, getHumidityFilter(NumericOperation.GREATER_OR_EQUAL, 60), true), + Arguments.of(60, getHumidityFilter(NumericOperation.GREATER_OR_EQUAL, 61), false), + Arguments.of(60, getHumidityFilter(NumericOperation.LESS, 61), true), + Arguments.of(60, getHumidityFilter(NumericOperation.LESS, 60), false), + Arguments.of(60, getHumidityFilter(NumericOperation.LESS_OR_EQUAL, 59), false), + Arguments.of(60, getHumidityFilter(NumericOperation.LESS_OR_EQUAL, 60), true) + ); + } + + @ParameterizedTest + @MethodSource("deviceHumidityFilters") + public void testFilterByDeviceHumidity(long humidityValue, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putTs(6, new LongDataPoint(System.currentTimeMillis(), humidityValue)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream deviceTemperatureAndHumidityFilters() { + return Stream.of(Arguments.of(22.8, 60, List.of(getTemperatureFilter(NumericOperation.EQUAL, 22.8), getHumidityFilter(NumericOperation.EQUAL, 60)), true), + Arguments.of(22.8, 60, List.of(getTemperatureFilter(NumericOperation.EQUAL, 22.8), getHumidityFilter(NumericOperation.GREATER_OR_EQUAL, 61)), false), + Arguments.of(22.8, 60, List.of(getTemperatureFilter(NumericOperation.GREATER, 23), getHumidityFilter(NumericOperation.GREATER_OR_EQUAL, 60)), false), + Arguments.of(22.8, 60, List.of(getTemperatureFilter(NumericOperation.GREATER_OR_EQUAL, 22.9), getHumidityFilter(NumericOperation.GREATER_OR_EQUAL, 61)), false) + ); + } + + @ParameterizedTest + @MethodSource("deviceTemperatureAndHumidityFilters") + public void testFilterByDeviceTemperatureAndHumidity(double tempValue, long humidityValue, List keyFilters, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putTs(5, new DoubleDataPoint(System.currentTimeMillis(), tempValue)); + deviceData.putTs(6, new LongDataPoint(System.currentTimeMillis(), humidityValue)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, keyFilters)).isEqualTo(result); + } + + private static Stream deviceVersionAttributeFilters() { + return Stream.of(Arguments.of(true, getActiveAttributeFilter(BooleanOperation.EQUAL, true), true), + Arguments.of(true, getActiveAttributeFilter(BooleanOperation.EQUAL, false), false), + Arguments.of(true, getActiveAttributeFilter(BooleanOperation.NOT_EQUAL, true), false), + Arguments.of(true, getActiveAttributeFilter(BooleanOperation.NOT_EQUAL, false), true) + ); + } + + @ParameterizedTest + @MethodSource("deviceVersionAttributeFilters") + public void testFilterByDeviceVersionAttribute(Boolean active, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putAttr(2, AttributeScope.SERVER_SCOPE, new BoolDataPoint(System.currentTimeMillis(), active)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream deviceActiveAndVersionFilters() { + return Stream.of(Arguments.of(true, "3.2.1", List.of(getActiveAttributeFilter(BooleanOperation.EQUAL, true), getVersionAttributeFilter(StringOperation.EQUAL, "3.2.1")), true), + Arguments.of(true, "3.2.1", List.of(getActiveAttributeFilter(BooleanOperation.EQUAL, true), getVersionAttributeFilter(StringOperation.EQUAL, "3.2.2")), false), + Arguments.of(true, "3.2.1", List.of(getActiveAttributeFilter(BooleanOperation.EQUAL, false), getVersionAttributeFilter(StringOperation.EQUAL, "3.2.1")), false), + Arguments.of(true, "3.2.1", List.of(getActiveAttributeFilter(BooleanOperation.EQUAL, false), getVersionAttributeFilter(StringOperation.EQUAL, "3.2.2")), false) + ); + } + + @ParameterizedTest + @MethodSource("deviceActiveAndVersionFilters") + public void testFilterByActiveAndVersionAttributes(Boolean active, String version, List keyFilters, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putAttr(1, AttributeScope.CLIENT_SCOPE, new StringDataPoint(System.currentTimeMillis(), version)); + deviceData.putAttr(2, AttributeScope.SERVER_SCOPE, new BoolDataPoint(System.currentTimeMillis(), active)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, keyFilters)).isEqualTo(result); + } + + private static EdqsFilter getVersionAttributeFilter(StringOperation operation, String predicateValue) { + StringFilterPredicate filterPredicate = new StringFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromString(predicateValue)); + + DataKey key = new DataKey(EntityKeyType.CLIENT_ATTRIBUTE, "version", 1); + return new EdqsFilter(key, EntityKeyValueType.STRING, filterPredicate); + } + + + private static EdqsFilter getActiveAttributeFilter(BooleanOperation operation, boolean predicateValue) { + BooleanFilterPredicate filterPredicate = new BooleanFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromBoolean(predicateValue)); + + DataKey key = new DataKey(EntityKeyType.SERVER_ATTRIBUTE, "active", 2); + return new EdqsFilter(key, EntityKeyValueType.BOOLEAN, filterPredicate); + } + + private static EdqsFilter getTemperatureFilter(NumericOperation operation, double predicateValue) { + return getTimeseriesFilter("temperature", 5, operation, predicateValue); + } + + private static EdqsFilter getHumidityFilter(NumericOperation operation, double predicateValue) { + return getTimeseriesFilter("humidity", 6, operation, predicateValue); + } + + private static EdqsFilter getTimeseriesFilter(String key, Integer keysId, NumericOperation operation, double predicateValue) { + NumericFilterPredicate filterPredicate = new NumericFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromDouble(predicateValue)); + + DataKey newKey = new DataKey(EntityKeyType.TIME_SERIES, key, keysId); + return new EdqsFilter(newKey, EntityKeyValueType.NUMERIC, filterPredicate); + } + + private static EdqsFilter getNameFilter(StringOperation operation, String predicateValue) { + return getStringEntityFieldFilter("name", operation, predicateValue); + } + + private static EdqsFilter getTypeFilter(StringOperation operation, String predicateValue) { + return getStringEntityFieldFilter("type", operation, predicateValue); + } + + private static EdqsFilter getStringEntityFieldFilter(String fieldName, StringOperation operation, String predicateValue) { + StringFilterPredicate filterPredicate = new StringFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromString(predicateValue)); + + DataKey key = new DataKey(EntityKeyType.ENTITY_FIELD, fieldName, 3); + return new EdqsFilter(key, EntityKeyValueType.STRING, filterPredicate); + } + + private static EdqsFilter getCreatedTimeFilter(NumericOperation operation, double predicateValue) { + return getDatetimeEntityFieldFilter("createdTime", operation, predicateValue); + } + + private static EdqsFilter getDatetimeEntityFieldFilter(String fieldName, NumericOperation operation, double predicateValue) { + NumericFilterPredicate filterPredicate = new NumericFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromDouble(predicateValue)); + + DataKey key = new DataKey(EntityKeyType.ENTITY_FIELD, fieldName, 3); + return new EdqsFilter(key, EntityKeyValueType.DATE_TIME, filterPredicate); + } + + private static EdqsFilter getComplexTemperatureFilter(NumericOperation operation, double predicateValue, ComplexOperation complexOperation, NumericOperation operation2, double predicateValue2) { + ComplexFilterPredicate complexFilterPredicate = getComplexNumericFilterPredicate(operation, predicateValue, complexOperation, operation2, predicateValue2); + + DataKey key = new DataKey(EntityKeyType.TIME_SERIES, "temperature", 5); + return new EdqsFilter(key, EntityKeyValueType.NUMERIC, complexFilterPredicate); + } + + private static EdqsFilter getComplexComplexTemperatureFilter(NumericOperation operation, double predicateValue, ComplexOperation complexOperation, NumericOperation operation2, double predicateValue2, + ComplexOperation complexOperation2, NumericOperation operation3, double predicateValue3) { + ComplexFilterPredicate complexFilterPredicate = getComplexNumericFilterPredicate(operation, predicateValue, complexOperation, operation2, predicateValue2); + + NumericFilterPredicate filterPredicate = new NumericFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromDouble(predicateValue)); + + ComplexFilterPredicate mainComplexFilterPredicate = new ComplexFilterPredicate(); + mainComplexFilterPredicate.setOperation(complexOperation2); + mainComplexFilterPredicate.setPredicates(List.of(complexFilterPredicate, filterPredicate)); + + DataKey key = new DataKey(EntityKeyType.TIME_SERIES, "temperature", 5); + return new EdqsFilter(key, EntityKeyValueType.NUMERIC, mainComplexFilterPredicate); + } + + private static ComplexFilterPredicate getComplexNumericFilterPredicate(NumericOperation operation, double predicateValue, ComplexOperation complexOperation, NumericOperation operation2, double predicateValue2) { + NumericFilterPredicate filterPredicate = new NumericFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromDouble(predicateValue)); + + NumericFilterPredicate filterPredicate2 = new NumericFilterPredicate(); + filterPredicate2.setOperation(operation2); + filterPredicate2.setValue(FilterPredicateValue.fromDouble(predicateValue2)); + + ComplexFilterPredicate complexFilterPredicate = new ComplexFilterPredicate(); + complexFilterPredicate.setOperation(complexOperation); + complexFilterPredicate.setPredicates(List.of(filterPredicate, filterPredicate2)); + return complexFilterPredicate; + } + + private static EdqsFilter getComplexComplexDeviceNameFilter(StringOperation operation, String predicateValue, ComplexOperation complexOperation, StringOperation operation2, String predicateValue2) { + ComplexFilterPredicate complexFilterPredicate = getComplexStringFilterPredicate(operation, predicateValue, complexOperation, operation2, predicateValue2); + DataKey key = new DataKey(EntityKeyType.ENTITY_FIELD, "name", 3); + return new EdqsFilter(key, EntityKeyValueType.STRING, complexFilterPredicate); + } + + private static EdqsFilter getComplexComplexDeviceNameFilter(StringOperation operation, String predicateValue, ComplexOperation complexOperation, StringOperation operation2, String predicateValue2, + ComplexOperation complexOperation2, StringOperation operation3, String predicateValue3) { + ComplexFilterPredicate complexFilterPredicate = getComplexStringFilterPredicate(operation, predicateValue, complexOperation, operation2, predicateValue2); + + StringFilterPredicate filterPredicate = new StringFilterPredicate(); + filterPredicate.setOperation(operation3); + filterPredicate.setValue(FilterPredicateValue.fromString(predicateValue3)); + + ComplexFilterPredicate mainComplexFilterPredicate = new ComplexFilterPredicate(); + mainComplexFilterPredicate.setOperation(complexOperation2); + mainComplexFilterPredicate.setPredicates(List.of(complexFilterPredicate, filterPredicate)); + + DataKey key = new DataKey(EntityKeyType.ENTITY_FIELD, "name", 3); + return new EdqsFilter(key, EntityKeyValueType.STRING, mainComplexFilterPredicate); + } + + private static ComplexFilterPredicate getComplexStringFilterPredicate(StringOperation operation, String predicateValue, ComplexOperation complexOperation, StringOperation operation2, String predicateValue2) { + StringFilterPredicate filterPredicate = new StringFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromString(predicateValue)); + + StringFilterPredicate filterPredicate2 = new StringFilterPredicate(); + filterPredicate2.setOperation(operation2); + filterPredicate2.setValue(FilterPredicateValue.fromString(predicateValue2)); + + ComplexFilterPredicate complexFilterPredicate = new ComplexFilterPredicate(); + complexFilterPredicate.setOperation(complexOperation); + complexFilterPredicate.setPredicates(List.of(filterPredicate, filterPredicate2)); + return complexFilterPredicate; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/SingleEntityFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/SingleEntityFilterTest.java new file mode 100644 index 0000000000..133816f576 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/SingleEntityFilterTest.java @@ -0,0 +1,133 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.SingleEntityFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; + +import java.util.Arrays; +import java.util.UUID; + +public class SingleEntityFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(new LatestTsKv(deviceId, new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, null, getEntityDataQuery(device.getId()), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + + result = repository.findEntityDataByQuery(tenantId, null, getEntityDataQuery(new DeviceId(UUID.randomUUID())), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, null, getEntityDataQuery(device.getId()), false); + Assert.assertEquals(1, result.getTotalElements()); + first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + @Test + public void testFindCustomerDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(new LatestTsKv(deviceId, new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, getEntityDataQuery(device.getId()), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, getEntityDataQuery(device.getId()), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + private static EntityDataQuery getEntityDataQuery(DeviceId deviceId) { + SingleEntityFilter filter = new SingleEntityFilter(); + filter.setSingleEntity(deviceId); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "state"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); + predicate.setValue(new FilterPredicateValue<>("LoRa-")); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, Arrays.asList(nameFilter)); + } + +} diff --git a/edqs/src/test/resources/edqs-test.properties b/edqs/src/test/resources/edqs-test.properties new file mode 100644 index 0000000000..8a041c7407 --- /dev/null +++ b/edqs/src/test/resources/edqs-test.properties @@ -0,0 +1,2 @@ +zk.enabled=false +service.type=edqs diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportHealthChecker.java b/monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportHealthChecker.java index 1b8a704fad..f5e6cb5469 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportHealthChecker.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/service/transport/TransportHealthChecker.java @@ -145,7 +145,7 @@ public abstract class TransportHealthChecker { TbResource newResource = ResourceUtils.getResource("lwm2m/resource.json", TbResource.class); diff --git a/monitoring/src/main/resources/lwm2m/device_profile.json b/monitoring/src/main/resources/lwm2m/device_profile.json index c75fe9ce7a..d23e02825d 100644 --- a/monitoring/src/main/resources/lwm2m/device_profile.json +++ b/monitoring/src/main/resources/lwm2m/device_profile.json @@ -12,14 +12,14 @@ "transportConfiguration": { "observeAttr": { "observe": [ - "/3_1.0/0/0" + "/3_1.1/0/0" ], "attribute": [], "telemetry": [ - "/3_1.0/0/0" + "/3_1.1/0/0" ], "keyName": { - "/3_1.0/0/0": "testData" + "/3_1.1/0/0": "testData" }, "attributeLwm2m": {} }, diff --git a/monitoring/src/main/resources/lwm2m/models/test-model.xml b/monitoring/src/main/resources/lwm2m/models/test-model.xml index 2d25f4698e..02b084127f 100644 --- a/monitoring/src/main/resources/lwm2m/models/test-model.xml +++ b/monitoring/src/main/resources/lwm2m/models/test-model.xml @@ -1,45 +1,331 @@ + - + + LwM2M Monitoring - - + 3 - urn:oma:lwm2m:oma:3 + urn:oma:lwm2m:oma:3:1.1 1.1 - 1.0 + 1.1 Single Mandatory - Test data + Manufacturer + R + Single + Optional + String + + + + + + Model Number R Single Optional String - + - + + Serial Number + R + Single + Optional + String + + + + + + Firmware Version + R + Single + Optional + String + + + + + + Reboot + E + Single + Mandatory + + + + + + + Factory Reset + E + Single + Optional + + + + + + + Available Power Sources + R + Multiple + Optional + Integer + 0..7 + + + + + Power Source Voltage + R + Multiple + Optional + Integer + + + + + + Power Source Current + R + Multiple + Optional + Integer + + + + + + Battery Level + R + Single + Optional + Integer + 0..100 + /100 + + + + Memory Free + R + Single + Optional + Integer + + + + + + Error Code + R + Multiple + Mandatory + Integer + 0..32 + + + + + Reset Error Code + E + Single + Optional + + + + + + + Current Time + RW + Single + Optional + Time + + + + + + UTC Offset + RW + Single + Optional + String + + + + + + Timezone + RW + Single + Optional + String + + + + + + Supported Binding and Modes + R + Single + Mandatory + String + + + + + Device Type + R + Single + Optional + String + + + + + Hardware Version + R + Single + Optional + String + + + + + Software Version + R + Single + Optional + String + + + + + Battery Status + R + Single + Optional + Integer + 0..6 + + + + Memory Total + R + Single + Optional + Integer + + + + + ExtDevInfo + R + Multiple + Optional + Objlnk + + + + diff --git a/monitoring/src/main/resources/lwm2m/resource.json b/monitoring/src/main/resources/lwm2m/resource.json index dcc0182bdd..bbef8da7ec 100644 --- a/monitoring/src/main/resources/lwm2m/resource.json +++ b/monitoring/src/main/resources/lwm2m/resource.json @@ -2,5 +2,5 @@ "title": "", "resourceType": "LWM2M_MODEL", "fileName": "test-model.xml", - "data": "PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPCEtLQoKICAgIENvcHlyaWdodCDCqSAyMDE2LTIwMjQgVGhlIFRoaW5nc2JvYXJkIEF1dGhvcnMKCiAgICBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKICAgIHlvdSBtYXkgbm90IHVzZSB0aGlzIGZpbGUgZXhjZXB0IGluIGNvbXBsaWFuY2Ugd2l0aCB0aGUgTGljZW5zZS4KICAgIFlvdSBtYXkgb2J0YWluIGEgY29weSBvZiB0aGUgTGljZW5zZSBhdAoKICAgICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKCiAgICBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiAgICBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAogICAgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiAgICBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiAgICBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi0tPgo8TFdNMk0geG1sbnM6eHNpPSJodHRwOi8vd3d3LnczLm9yZy8yMDAxL1hNTFNjaGVtYS1pbnN0YW5jZSIKICAgICAgIHhzaTpub05hbWVzcGFjZVNjaGVtYUxvY2F0aW9uPSJodHRwOi8vd3d3Lm9wZW5tb2JpbGVhbGxpYW5jZS5vcmcvdGVjaC9wcm9maWxlcy9MV00yTS12MV8xLnhzZCI+CiAgICA8T2JqZWN0IE9iamVjdFR5cGU9Ik1PRGVmaW5pdGlvbiI+CiAgICAgICAgPE5hbWU+THdNMk0gTW9uaXRvcmluZzwvTmFtZT4KICAgICAgICA8RGVzY3JpcHRpb24xPgogICAgICAgICAgICA8IVtDREFUQVtdXT48L0Rlc2NyaXB0aW9uMT4KICAgICAgICA8T2JqZWN0SUQ+MzwvT2JqZWN0SUQ+CiAgICAgICAgPE9iamVjdFVSTj51cm46b21hOmx3bTJtOm9tYTozPC9PYmplY3RVUk4+CiAgICAgICAgPExXTTJNVmVyc2lvbj4xLjE8L0xXTTJNVmVyc2lvbj4KICAgICAgICA8T2JqZWN0VmVyc2lvbj4xLjA8L09iamVjdFZlcnNpb24+CiAgICAgICAgPE11bHRpcGxlSW5zdGFuY2VzPlNpbmdsZTwvTXVsdGlwbGVJbnN0YW5jZXM+CiAgICAgICAgPE1hbmRhdG9yeT5NYW5kYXRvcnk8L01hbmRhdG9yeT4KICAgICAgICA8UmVzb3VyY2VzPgogICAgICAgICAgICA8SXRlbSBJRD0iMCI+CiAgICAgICAgICAgICAgICA8TmFtZT5UZXN0IGRhdGE8L05hbWU+CiAgICAgICAgICAgICAgICA8T3BlcmF0aW9ucz5SPC9PcGVyYXRpb25zPgogICAgICAgICAgICAgICAgPE11bHRpcGxlSW5zdGFuY2VzPlNpbmdsZTwvTXVsdGlwbGVJbnN0YW5jZXM+CiAgICAgICAgICAgICAgICA8TWFuZGF0b3J5Pk9wdGlvbmFsPC9NYW5kYXRvcnk+CiAgICAgICAgICAgICAgICA8VHlwZT5TdHJpbmc8L1R5cGU+CiAgICAgICAgICAgICAgICA8UmFuZ2VFbnVtZXJhdGlvbj48L1JhbmdlRW51bWVyYXRpb24+CiAgICAgICAgICAgICAgICA8VW5pdHM+PC9Vbml0cz4KICAgICAgICAgICAgICAgIDxEZXNjcmlwdGlvbj48IVtDREFUQVtUZXN0IGRhdGFdXT48L0Rlc2NyaXB0aW9uPgogICAgICAgICAgICA8L0l0ZW0+CiAgICAgICAgPC9SZXNvdXJjZXM+CiAgICAgICAgPERlc2NyaXB0aW9uMj48L0Rlc2NyaXB0aW9uMj4KICAgIDwvT2JqZWN0Pgo8L0xXTTJNPgo=" + "data": "PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KCjwhLS0KRklMRSBJTkZPUk1BVElPTgoKT01BIFBlcm1hbmVudCBEb2N1bWVudAogICBGaWxlOiBPTUEtU1VQLVhNTF8zLVYxXzItMjAyMDExMTAtQS54bWwKICAgUGF0aDogaHR0cDovL3d3dy5vcGVubW9iaWxlYWxsaWFuY2Uub3JnL3JlbGVhc2UvT2JqTHdNMk1fRGV2aWNlLwoKT01OQSBMd00yTSBSZWdpc3RyeQogICBQYXRoOiBodHRwczovL2dpdGh1Yi5jb20vT3Blbk1vYmlsZUFsbGlhbmNlL2x3bTJtLXJlZ2lzdHJ5CiAgIE5hbWU6IDMueG1sCgpOT1JNQVRJVkUgSU5GT1JNQVRJT04KCiAgSW5mb3JtYXRpb24gYWJvdXQgdGhpcyBmaWxlIGNhbiBiZSBmb3VuZCBpbiB0aGUgbGF0ZXN0IHJldmlzaW9uIG9mCgogICAgT01BLVRTLUxpZ2h0d2VpZ2h0TTJNX0NvcmUtVjFfMgoKICBUaGlzIGlzIGF2YWlsYWJsZSBhdCBodHRwOi8vd3d3Lm9wZW5tb2JpbGVhbGxpYW5jZS5vcmcvcmVsZWFzZS9MaWdodHdlaWdodE0yTS8KCiAgU2VuZCBjb21tZW50cyB0byBodHRwczovL2dpdGh1Yi5jb20vT3Blbk1vYmlsZUFsbGlhbmNlL09NQV9Md00yTV9mb3JfRGV2ZWxvcGVycy9pc3N1ZXMKCkxFR0FMIERJU0NMQUlNRVIKCiAgQ29weXJpZ2h0IDIwMjAgT3BlbiBNb2JpbGUgQWxsaWFuY2UuCgogIFJlZGlzdHJpYnV0aW9uIGFuZCB1c2UgaW4gc291cmNlIGFuZCBiaW5hcnkgZm9ybXMsIHdpdGggb3Igd2l0aG91dAogIG1vZGlmaWNhdGlvbiwgYXJlIHBlcm1pdHRlZCBwcm92aWRlZCB0aGF0IHRoZSBmb2xsb3dpbmcgY29uZGl0aW9ucwogIGFyZSBtZXQ6CgogIDEuIFJlZGlzdHJpYnV0aW9ucyBvZiBzb3VyY2UgY29kZSBtdXN0IHJldGFpbiB0aGUgYWJvdmUgY29weXJpZ2h0CiAgbm90aWNlLCB0aGlzIGxpc3Qgb2YgY29uZGl0aW9ucyBhbmQgdGhlIGZvbGxvd2luZyBkaXNjbGFpbWVyLgogIDIuIFJlZGlzdHJpYnV0aW9ucyBpbiBiaW5hcnkgZm9ybSBtdXN0IHJlcHJvZHVjZSB0aGUgYWJvdmUgY29weXJpZ2h0CiAgbm90aWNlLCB0aGlzIGxpc3Qgb2YgY29uZGl0aW9ucyBhbmQgdGhlIGZvbGxvd2luZyBkaXNjbGFpbWVyIGluIHRoZQogIGRvY3VtZW50YXRpb24gYW5kL29yIG90aGVyIG1hdGVyaWFscyBwcm92aWRlZCB3aXRoIHRoZSBkaXN0cmlidXRpb24uCiAgMy4gTmVpdGhlciB0aGUgbmFtZSBvZiB0aGUgY29weXJpZ2h0IGhvbGRlciBub3IgdGhlIG5hbWVzIG9mIGl0cwogIGNvbnRyaWJ1dG9ycyBtYXkgYmUgdXNlZCB0byBlbmRvcnNlIG9yIHByb21vdGUgcHJvZHVjdHMgZGVyaXZlZAogIGZyb20gdGhpcyBzb2Z0d2FyZSB3aXRob3V0IHNwZWNpZmljIHByaW9yIHdyaXR0ZW4gcGVybWlzc2lvbi4KCiAgVEhJUyBTT0ZUV0FSRSBJUyBQUk9WSURFRCBCWSBUSEUgQ09QWVJJR0hUIEhPTERFUlMgQU5EIENPTlRSSUJVVE9SUwogICJBUyBJUyIgQU5EIEFOWSBFWFBSRVNTIE9SIElNUExJRUQgV0FSUkFOVElFUywgSU5DTFVESU5HLCBCVVQgTk9UCiAgTElNSVRFRCBUTywgVEhFIElNUExJRUQgV0FSUkFOVElFUyBPRiBNRVJDSEFOVEFCSUxJVFkgQU5EIEZJVE5FU1MKICBGT1IgQSBQQVJUSUNVTEFSIFBVUlBPU0UgQVJFIERJU0NMQUlNRUQuIElOIE5PIEVWRU5UIFNIQUxMIFRIRQogIENPUFlSSUdIVCBIT0xERVIgT1IgQ09OVFJJQlVUT1JTIEJFIExJQUJMRSBGT1IgQU5ZIERJUkVDVCwgSU5ESVJFQ1QsCiAgSU5DSURFTlRBTCwgU1BFQ0lBTCwgRVhFTVBMQVJZLCBPUiBDT05TRVFVRU5USUFMIERBTUFHRVMgKElOQ0xVRElORywKICBCVVQgTk9UIExJTUlURUQgVE8sIFBST0NVUkVNRU5UIE9GIFNVQlNUSVRVVEUgR09PRFMgT1IgU0VSVklDRVM7CiAgTE9TUyBPRiBVU0UsIERBVEEsIE9SIFBST0ZJVFM7IE9SIEJVU0lORVNTIElOVEVSUlVQVElPTikgSE9XRVZFUgogIENBVVNFRCBBTkQgT04gQU5ZIFRIRU9SWSBPRiBMSUFCSUxJVFksIFdIRVRIRVIgSU4gQ09OVFJBQ1QsIFNUUklDVAogIExJQUJJTElUWSwgT1IgVE9SVCAoSU5DTFVESU5HIE5FR0xJR0VOQ0UgT1IgT1RIRVJXSVNFKSBBUklTSU5HIElOCiAgQU5ZIFdBWSBPVVQgT0YgVEhFIFVTRSBPRiBUSElTIFNPRlRXQVJFLCBFVkVOIElGIEFEVklTRUQgT0YgVEhFCiAgUE9TU0lCSUxJVFkgT0YgU1VDSCBEQU1BR0UuCgogIFRoZSBhYm92ZSBsaWNlbnNlIGlzIHVzZWQgYXMgYSBsaWNlbnNlIHVuZGVyIGNvcHlyaWdodCBvbmx5LiAgUGxlYXNlCiAgcmVmZXJlbmNlIHRoZSBPTUEgSVBSIFBvbGljeSBmb3IgcGF0ZW50IGxpY2Vuc2luZyB0ZXJtczoKICBodHRwczovL3d3dy5vbWFzcGVjd29ya3Mub3JnL2Fib3V0L2ludGVsbGVjdHVhbC1wcm9wZXJ0eS1yaWdodHMvCgotLT4KCjxMV00yTSB4bWxuczp4c2k9Imh0dHA6Ly93d3cudzMub3JnLzIwMDEvWE1MU2NoZW1hLWluc3RhbmNlIiB4c2k6bm9OYW1lc3BhY2VTY2hlbWFMb2NhdGlvbj0iaHR0cDovL3d3dy5vcGVubW9iaWxlYWxsaWFuY2Uub3JnL3RlY2gvcHJvZmlsZXMvTFdNMk0tdjFfMS54c2QiPgogICAgPE9iamVjdCBPYmplY3RUeXBlPSJNT0RlZmluaXRpb24iPgogICAgICAgIDxOYW1lPkx3TTJNIE1vbml0b3Jpbmc8L05hbWU+CiAgICAgICAgPERlc2NyaXB0aW9uMT48IVtDREFUQVtUaGlzIEx3TTJNIE9iamVjdCBwcm92aWRlcyBhIHJhbmdlIG9mIGRldmljZSByZWxhdGVkIGluZm9ybWF0aW9uIHdoaWNoIGNhbiBiZSBxdWVyaWVkIGJ5IHRoZSBMd00yTSBTZXJ2ZXIsIGFuZCBhIGRldmljZSByZWJvb3QgYW5kIGZhY3RvcnkgcmVzZXQgZnVuY3Rpb24uXV0+PC9EZXNjcmlwdGlvbjE+CiAgICAgICAgPE9iamVjdElEPjM8L09iamVjdElEPgogICAgICAgIDxPYmplY3RVUk4+dXJuOm9tYTpsd20ybTpvbWE6MzoxLjE8L09iamVjdFVSTj4KICAgICAgICA8TFdNMk1WZXJzaW9uPjEuMTwvTFdNMk1WZXJzaW9uPgogICAgICAgIDxPYmplY3RWZXJzaW9uPjEuMTwvT2JqZWN0VmVyc2lvbj4KICAgICAgICA8TXVsdGlwbGVJbnN0YW5jZXM+U2luZ2xlPC9NdWx0aXBsZUluc3RhbmNlcz4KICAgICAgICA8TWFuZGF0b3J5Pk1hbmRhdG9yeTwvTWFuZGF0b3J5PgogICAgICAgIDxSZXNvdXJjZXM+CiAgICAgICAgICAgIDxJdGVtIElEPSIwIj4KICAgICAgICAgICAgICAgIDxOYW1lPk1hbnVmYWN0dXJlcjwvTmFtZT4KICAgICAgICAgICAgICAgIDxPcGVyYXRpb25zPlI8L09wZXJhdGlvbnM+CiAgICAgICAgICAgICAgICA8TXVsdGlwbGVJbnN0YW5jZXM+U2luZ2xlPC9NdWx0aXBsZUluc3RhbmNlcz4KICAgICAgICAgICAgICAgIDxNYW5kYXRvcnk+T3B0aW9uYWw8L01hbmRhdG9yeT4KICAgICAgICAgICAgICAgIDxUeXBlPlN0cmluZzwvVHlwZT4KICAgICAgICAgICAgICAgIDxSYW5nZUVudW1lcmF0aW9uPjwvUmFuZ2VFbnVtZXJhdGlvbj4KICAgICAgICAgICAgICAgIDxVbml0cz48L1VuaXRzPgogICAgICAgICAgICAgICAgPERlc2NyaXB0aW9uPjwhW0NEQVRBW0h1bWFuIHJlYWRhYmxlIG1hbnVmYWN0dXJlciBuYW1lXV0+PC9EZXNjcmlwdGlvbj4KICAgICAgICAgICAgPC9JdGVtPgogICAgICAgICAgICA8SXRlbSBJRD0iMSI+CiAgICAgICAgICAgICAgICA8TmFtZT5Nb2RlbCBOdW1iZXI8L05hbWU+CiAgICAgICAgICAgICAgICA8T3BlcmF0aW9ucz5SPC9PcGVyYXRpb25zPgogICAgICAgICAgICAgICAgPE11bHRpcGxlSW5zdGFuY2VzPlNpbmdsZTwvTXVsdGlwbGVJbnN0YW5jZXM+CiAgICAgICAgICAgICAgICA8TWFuZGF0b3J5Pk9wdGlvbmFsPC9NYW5kYXRvcnk+CiAgICAgICAgICAgICAgICA8VHlwZT5TdHJpbmc8L1R5cGU+CiAgICAgICAgICAgICAgICA8UmFuZ2VFbnVtZXJhdGlvbj48L1JhbmdlRW51bWVyYXRpb24+CiAgICAgICAgICAgICAgICA8VW5pdHM+PC9Vbml0cz4KICAgICAgICAgICAgICAgIDxEZXNjcmlwdGlvbj48IVtDREFUQVtBIG1vZGVsIGlkZW50aWZpZXIgKG1hbnVmYWN0dXJlciBzcGVjaWZpZWQgc3RyaW5nKV1dPjwvRGVzY3JpcHRpb24+CiAgICAgICAgICAgIDwvSXRlbT4KICAgICAgICAgICAgPEl0ZW0gSUQ9IjIiPgogICAgICAgICAgICAgICAgPE5hbWU+U2VyaWFsIE51bWJlcjwvTmFtZT4KICAgICAgICAgICAgICAgIDxPcGVyYXRpb25zPlI8L09wZXJhdGlvbnM+CiAgICAgICAgICAgICAgICA8TXVsdGlwbGVJbnN0YW5jZXM+U2luZ2xlPC9NdWx0aXBsZUluc3RhbmNlcz4KICAgICAgICAgICAgICAgIDxNYW5kYXRvcnk+T3B0aW9uYWw8L01hbmRhdG9yeT4KICAgICAgICAgICAgICAgIDxUeXBlPlN0cmluZzwvVHlwZT4KICAgICAgICAgICAgICAgIDxSYW5nZUVudW1lcmF0aW9uPjwvUmFuZ2VFbnVtZXJhdGlvbj4KICAgICAgICAgICAgICAgIDxVbml0cz48L1VuaXRzPgogICAgICAgICAgICAgICAgPERlc2NyaXB0aW9uPjwhW0NEQVRBW1NlcmlhbCBOdW1iZXJdXT48L0Rlc2NyaXB0aW9uPgogICAgICAgICAgICA8L0l0ZW0+CiAgICAgICAgICAgIDxJdGVtIElEPSIzIj4KICAgICAgICAgICAgICAgIDxOYW1lPkZpcm13YXJlIFZlcnNpb248L05hbWU+CiAgICAgICAgICAgICAgICA8T3BlcmF0aW9ucz5SPC9PcGVyYXRpb25zPgogICAgICAgICAgICAgICAgPE11bHRpcGxlSW5zdGFuY2VzPlNpbmdsZTwvTXVsdGlwbGVJbnN0YW5jZXM+CiAgICAgICAgICAgICAgICA8TWFuZGF0b3J5Pk9wdGlvbmFsPC9NYW5kYXRvcnk+CiAgICAgICAgICAgICAgICA8VHlwZT5TdHJpbmc8L1R5cGU+CiAgICAgICAgICAgICAgICA8UmFuZ2VFbnVtZXJhdGlvbj48L1JhbmdlRW51bWVyYXRpb24+CiAgICAgICAgICAgICAgICA8VW5pdHM+PC9Vbml0cz4KICAgICAgICAgICAgICAgIDxEZXNjcmlwdGlvbj48IVtDREFUQVtDdXJyZW50IGZpcm13YXJlIHZlcnNpb24gb2YgdGhlIERldmljZS5UaGUgRmlybXdhcmUgTWFuYWdlbWVudCBmdW5jdGlvbiBjb3VsZCByZWx5IG9uIHRoaXMgcmVzb3VyY2UuXV0+PC9EZXNjcmlwdGlvbj4KICAgICAgICAgICAgPC9JdGVtPgogICAgICAgICAgICA8SXRlbSBJRD0iNCI+CiAgICAgICAgICAgICAgICA8TmFtZT5SZWJvb3Q8L05hbWU+CiAgICAgICAgICAgICAgICA8T3BlcmF0aW9ucz5FPC9PcGVyYXRpb25zPgogICAgICAgICAgICAgICAgPE11bHRpcGxlSW5zdGFuY2VzPlNpbmdsZTwvTXVsdGlwbGVJbnN0YW5jZXM+CiAgICAgICAgICAgICAgICA8TWFuZGF0b3J5Pk1hbmRhdG9yeTwvTWFuZGF0b3J5PgogICAgICAgICAgICAgICAgPFR5cGU+PC9UeXBlPgogICAgICAgICAgICAgICAgPFJhbmdlRW51bWVyYXRpb24+PC9SYW5nZUVudW1lcmF0aW9uPgogICAgICAgICAgICAgICAgPFVuaXRzPjwvVW5pdHM+CiAgICAgICAgICAgICAgICA8RGVzY3JpcHRpb24+PCFbQ0RBVEFbUmVib290IHRoZSBMd00yTSBEZXZpY2UgdG8gcmVzdG9yZSB0aGUgRGV2aWNlIGZyb20gdW5leHBlY3RlZCBmaXJtd2FyZSBmYWlsdXJlLl1dPjwvRGVzY3JpcHRpb24+CiAgICAgICAgICAgIDwvSXRlbT4KICAgICAgICAgICAgPEl0ZW0gSUQ9IjUiPgogICAgICAgICAgICAgICAgPE5hbWU+RmFjdG9yeSBSZXNldDwvTmFtZT4KICAgICAgICAgICAgICAgIDxPcGVyYXRpb25zPkU8L09wZXJhdGlvbnM+CiAgICAgICAgICAgICAgICA8TXVsdGlwbGVJbnN0YW5jZXM+U2luZ2xlPC9NdWx0aXBsZUluc3RhbmNlcz4KICAgICAgICAgICAgICAgIDxNYW5kYXRvcnk+T3B0aW9uYWw8L01hbmRhdG9yeT4KICAgICAgICAgICAgICAgIDxUeXBlPjwvVHlwZT4KICAgICAgICAgICAgICAgIDxSYW5nZUVudW1lcmF0aW9uPjwvUmFuZ2VFbnVtZXJhdGlvbj4KICAgICAgICAgICAgICAgIDxVbml0cz48L1VuaXRzPgogICAgICAgICAgICAgICAgPERlc2NyaXB0aW9uPjwhW0NEQVRBW1BlcmZvcm0gZmFjdG9yeSByZXNldCBvZiB0aGUgTHdNMk0gRGV2aWNlIHRvIG1ha2UgdGhlIEx3TTJNIERldmljZSB0byBnbyB0aHJvdWdoIGluaXRpYWwgZGVwbG95bWVudCBzZXF1ZW5jZSB3aGVyZSBwcm92aXNpb25pbmcgYW5kIGJvb3RzdHJhcCBzZXF1ZW5jZSBpcyBwZXJmb3JtZWQuIFRoaXMgcmVxdWlyZXMgY2xpZW50IGVuc3VyaW5nIHBvc3QgZmFjdG9yeSByZXNldCB0byBoYXZlIG1pbmltYWwgaW5mb3JtYXRpb24gdG8gYWxsb3cgaXQgdG8gY2Fycnkgb3V0IG9uZSBvZiB0aGUgYm9vdHN0cmFwIG1ldGhvZHMgc3BlY2lmaWVkIGluIHNlY3Rpb24gNS4yLjMuCldoZW4gdGhpcyBSZXNvdXJjZSBpcyBleGVjdXRlZCwgIkRlLXJlZ2lzdGVyIiBvcGVyYXRpb24gTUFZIGJlIHNlbnQgdG8gdGhlIEx3TTJNIFNlcnZlcihzKSBiZWZvcmUgZmFjdG9yeSByZXNldCBvZiB0aGUgTHdNMk0gRGV2aWNlLl1dPjwvRGVzY3JpcHRpb24+CiAgICAgICAgICAgIDwvSXRlbT4KICAgICAgICAgICAgPEl0ZW0gSUQ9IjYiPgogICAgICAgICAgICAgICAgPE5hbWU+QXZhaWxhYmxlIFBvd2VyIFNvdXJjZXM8L05hbWU+CiAgICAgICAgICAgICAgICA8T3BlcmF0aW9ucz5SPC9PcGVyYXRpb25zPgogICAgICAgICAgICAgICAgPE11bHRpcGxlSW5zdGFuY2VzPk11bHRpcGxlPC9NdWx0aXBsZUluc3RhbmNlcz4KICAgICAgICAgICAgICAgIDxNYW5kYXRvcnk+T3B0aW9uYWw8L01hbmRhdG9yeT4KICAgICAgICAgICAgICAgIDxUeXBlPkludGVnZXI8L1R5cGU+CiAgICAgICAgICAgICAgICA8UmFuZ2VFbnVtZXJhdGlvbj4wLi43PC9SYW5nZUVudW1lcmF0aW9uPgogICAgICAgICAgICAgICAgPFVuaXRzPjwvVW5pdHM+CiAgICAgICAgICAgICAgICA8RGVzY3JpcHRpb24+PCFbQ0RBVEFbMDogREMgcG93ZXIKMTogSW50ZXJuYWwgQmF0dGVyeQoyOiBFeHRlcm5hbCBCYXR0ZXJ5CjM6IEZ1ZWwgQ2VsbAo0OiBQb3dlciBvdmVyIEV0aGVybmV0CjU6IFVTQgo2OiBBQyAoTWFpbnMpIHBvd2VyCjc6IFNvbGFyClRoZSBzYW1lIFJlc291cmNlIEluc3RhbmNlIElEIE1VU1QgYmUgdXNlZCB0byBhc3NvY2lhdGUgYSBnaXZlbiBQb3dlciBTb3VyY2UgKFJlc291cmNlIElEOjYpIHdpdGggaXRzIFByZXNlbnQgVm9sdGFnZSAoUmVzb3VyY2UgSUQ6NykgYW5kIGl0cyBQcmVzZW50IEN1cnJlbnQgKFJlc291cmNlIElEOjgpXV0+PC9EZXNjcmlwdGlvbj4KICAgICAgICAgICAgPC9JdGVtPgogICAgICAgICAgICA8SXRlbSBJRD0iNyI+CiAgICAgICAgICAgICAgICA8TmFtZT5Qb3dlciBTb3VyY2UgVm9sdGFnZTwvTmFtZT4KICAgICAgICAgICAgICAgIDxPcGVyYXRpb25zPlI8L09wZXJhdGlvbnM+CiAgICAgICAgICAgICAgICA8TXVsdGlwbGVJbnN0YW5jZXM+TXVsdGlwbGU8L011bHRpcGxlSW5zdGFuY2VzPgogICAgICAgICAgICAgICAgPE1hbmRhdG9yeT5PcHRpb25hbDwvTWFuZGF0b3J5PgogICAgICAgICAgICAgICAgPFR5cGU+SW50ZWdlcjwvVHlwZT4KICAgICAgICAgICAgICAgIDxSYW5nZUVudW1lcmF0aW9uPjwvUmFuZ2VFbnVtZXJhdGlvbj4KICAgICAgICAgICAgICAgIDxVbml0cz48L1VuaXRzPgogICAgICAgICAgICAgICAgPERlc2NyaXB0aW9uPjwhW0NEQVRBW1ByZXNlbnQgdm9sdGFnZSBmb3IgZWFjaCBBdmFpbGFibGUgUG93ZXIgU291cmNlcyBSZXNvdXJjZSBJbnN0YW5jZS4gVGhlIHVuaXQgdXNlZCBmb3IgdGhpcyByZXNvdXJjZSBpcyBpbiBtVi5dXT48L0Rlc2NyaXB0aW9uPgogICAgICAgICAgICA8L0l0ZW0+CiAgICAgICAgICAgIDxJdGVtIElEPSI4Ij4KICAgICAgICAgICAgICAgIDxOYW1lPlBvd2VyIFNvdXJjZSBDdXJyZW50PC9OYW1lPgogICAgICAgICAgICAgICAgPE9wZXJhdGlvbnM+UjwvT3BlcmF0aW9ucz4KICAgICAgICAgICAgICAgIDxNdWx0aXBsZUluc3RhbmNlcz5NdWx0aXBsZTwvTXVsdGlwbGVJbnN0YW5jZXM+CiAgICAgICAgICAgICAgICA8TWFuZGF0b3J5Pk9wdGlvbmFsPC9NYW5kYXRvcnk+CiAgICAgICAgICAgICAgICA8VHlwZT5JbnRlZ2VyPC9UeXBlPgogICAgICAgICAgICAgICAgPFJhbmdlRW51bWVyYXRpb24+PC9SYW5nZUVudW1lcmF0aW9uPgogICAgICAgICAgICAgICAgPFVuaXRzPjwvVW5pdHM+CiAgICAgICAgICAgICAgICA8RGVzY3JpcHRpb24+PCFbQ0RBVEFbUHJlc2VudCBjdXJyZW50IGZvciBlYWNoIEF2YWlsYWJsZSBQb3dlciBTb3VyY2UuIFRoZSB1bml0IHVzZWQgZm9yIHRoaXMgcmVzb3VyY2UgaXMgaW4gbUEuXV0+PC9EZXNjcmlwdGlvbj4KICAgICAgICAgICAgPC9JdGVtPgogICAgICAgICAgICA8SXRlbSBJRD0iOSI+CiAgICAgICAgICAgICAgICA8TmFtZT5CYXR0ZXJ5IExldmVsPC9OYW1lPgogICAgICAgICAgICAgICAgPE9wZXJhdGlvbnM+UjwvT3BlcmF0aW9ucz4KICAgICAgICAgICAgICAgIDxNdWx0aXBsZUluc3RhbmNlcz5TaW5nbGU8L011bHRpcGxlSW5zdGFuY2VzPgogICAgICAgICAgICAgICAgPE1hbmRhdG9yeT5PcHRpb25hbDwvTWFuZGF0b3J5PgogICAgICAgICAgICAgICAgPFR5cGU+SW50ZWdlcjwvVHlwZT4KICAgICAgICAgICAgICAgIDxSYW5nZUVudW1lcmF0aW9uPjAuLjEwMDwvUmFuZ2VFbnVtZXJhdGlvbj4KICAgICAgICAgICAgICAgIDxVbml0cz4vMTAwPC9Vbml0cz4KICAgICAgICAgICAgICAgIDxEZXNjcmlwdGlvbj48IVtDREFUQVtDb250YWlucyB0aGUgY3VycmVudCBiYXR0ZXJ5IGxldmVsIGFzIGEgcGVyY2VudGFnZSAod2l0aCBhIHJhbmdlIGZyb20gMCB0byAxMDApLiBUaGlzIHZhbHVlIGlzIG9ubHkgdmFsaWQgZm9yIHRoZSBEZXZpY2UgaW50ZXJuYWwgQmF0dGVyeSBpZiBwcmVzZW50IChvbmUgQXZhaWxhYmxlIFBvd2VyIFNvdXJjZXMgUmVzb3VyY2UgSW5zdGFuY2UgaXMgMSkuXV0+PC9EZXNjcmlwdGlvbj4KICAgICAgICAgICAgPC9JdGVtPgogICAgICAgICAgICA8SXRlbSBJRD0iMTAiPgogICAgICAgICAgICAgICAgPE5hbWU+TWVtb3J5IEZyZWU8L05hbWU+CiAgICAgICAgICAgICAgICA8T3BlcmF0aW9ucz5SPC9PcGVyYXRpb25zPgogICAgICAgICAgICAgICAgPE11bHRpcGxlSW5zdGFuY2VzPlNpbmdsZTwvTXVsdGlwbGVJbnN0YW5jZXM+CiAgICAgICAgICAgICAgICA8TWFuZGF0b3J5Pk9wdGlvbmFsPC9NYW5kYXRvcnk+CiAgICAgICAgICAgICAgICA8VHlwZT5JbnRlZ2VyPC9UeXBlPgogICAgICAgICAgICAgICAgPFJhbmdlRW51bWVyYXRpb24+PC9SYW5nZUVudW1lcmF0aW9uPgogICAgICAgICAgICAgICAgPFVuaXRzPjwvVW5pdHM+CiAgICAgICAgICAgICAgICA8RGVzY3JpcHRpb24+PCFbQ0RBVEFbRXN0aW1hdGVkIGN1cnJlbnQgYXZhaWxhYmxlIGFtb3VudCBvZiBzdG9yYWdlIHNwYWNlIHdoaWNoIGNhbiBzdG9yZSBkYXRhIGFuZCBzb2Z0d2FyZSBpbiB0aGUgTHdNMk0gRGV2aWNlIChleHByZXNzZWQgaW4ga2lsb2J5dGVzKS4gTm90ZTogMSBraWxvYnl0ZSBjb3JyZXNwb25kcyB0byAxMDAwIGJ5dGVzLl1dPjwvRGVzY3JpcHRpb24+CiAgICAgICAgICAgIDwvSXRlbT4KICAgICAgICAgICAgPEl0ZW0gSUQ9IjExIj4KICAgICAgICAgICAgICAgIDxOYW1lPkVycm9yIENvZGU8L05hbWU+CiAgICAgICAgICAgICAgICA8T3BlcmF0aW9ucz5SPC9PcGVyYXRpb25zPgogICAgICAgICAgICAgICAgPE11bHRpcGxlSW5zdGFuY2VzPk11bHRpcGxlPC9NdWx0aXBsZUluc3RhbmNlcz4KICAgICAgICAgICAgICAgIDxNYW5kYXRvcnk+TWFuZGF0b3J5PC9NYW5kYXRvcnk+CiAgICAgICAgICAgICAgICA8VHlwZT5JbnRlZ2VyPC9UeXBlPgogICAgICAgICAgICAgICAgPFJhbmdlRW51bWVyYXRpb24+MC4uMzI8L1JhbmdlRW51bWVyYXRpb24+CiAgICAgICAgICAgICAgICA8VW5pdHM+PC9Vbml0cz4KICAgICAgICAgICAgICAgIDxEZXNjcmlwdGlvbj48IVtDREFUQVswPU5vIGVycm9yCjE9TG93IGJhdHRlcnkgcG93ZXIKMj1FeHRlcm5hbCBwb3dlciBzdXBwbHkgb2ZmCjM9R1BTIG1vZHVsZSBmYWlsdXJlCjQ9TG93IHJlY2VpdmVkIHNpZ25hbCBzdHJlbmd0aAo1PU91dCBvZiBtZW1vcnkKNj1TTVMgZmFpbHVyZQo3PUlQIGNvbm5lY3Rpdml0eSBmYWlsdXJlCjg9UGVyaXBoZXJhbCBtYWxmdW5jdGlvbgo5Li4xNT1SZXNlcnZlZCBmb3IgZnV0dXJlIHVzZQoxNi4uMzI9RGV2aWNlIHNwZWNpZmljIGVycm9yIGNvZGVzCgpXaGVuIHRoZSBzaW5nbGUgRGV2aWNlIE9iamVjdCBJbnN0YW5jZSBpcyBpbml0aWF0ZWQsIHRoZXJlIGlzIG9ubHkgb25lIGVycm9yIGNvZGUgUmVzb3VyY2UgSW5zdGFuY2Ugd2hvc2UgdmFsdWUgaXMgZXF1YWwgdG8gMCB0aGF0IG1lYW5zIG5vIGVycm9yLiBXaGVuIHRoZSBmaXJzdCBlcnJvciBoYXBwZW5zLCB0aGUgTHdNMk0gQ2xpZW50IGNoYW5nZXMgZXJyb3IgY29kZSBSZXNvdXJjZSBJbnN0YW5jZSB0byBhbnkgbm9uLXplcm8gdmFsdWUgdG8gaW5kaWNhdGUgdGhlIGVycm9yIHR5cGUuIFdoZW4gYW55IG90aGVyIGVycm9yIGhhcHBlbnMsIGEgbmV3IGVycm9yIGNvZGUgUmVzb3VyY2UgSW5zdGFuY2UgaXMgY3JlYXRlZC4gV2hlbiBhbiBlcnJvciBhc3NvY2lhdGVkIHdpdGggYSBSZXNvdXJjZSBJbnN0YW5jZSBpcyBubyBsb25nZXIgcHJlc2VudCwgdGhhdCBSZXNvdXJjZSBJbnN0YW5jZSBpcyBkZWxldGVkLiBXaGVuIHRoZSBzaW5nbGUgZXhpc3RpbmcgZXJyb3IgaXMgbm8gbG9uZ2VyIHByZXNlbnQsIHRoZSBMd00yTSBDbGllbnQgcmV0dXJucyB0byB0aGUgb3JpZ2luYWwgbm8gZXJyb3Igc3RhdGUgd2hlcmUgSW5zdGFuY2UgMCBoYXMgdmFsdWUgMC4KVGhpcyBlcnJvciBjb2RlIFJlc291cmNlIE1BWSBiZSBvYnNlcnZlZCBieSB0aGUgTHdNMk0gU2VydmVyLiBIb3cgdG8gZGVhbCB3aXRoIEx3TTJNIENsaWVudOKAmXMgZXJyb3IgcmVwb3J0IGRlcGVuZHMgb24gdGhlIHBvbGljeSBvZiB0aGUgTHdNMk0gU2VydmVyLiBFcnJvciBjb2RlcyBpbiBiZXR3ZWVuIDE2IGFuZCAzMiBhcmUgc3BlY2lmaWMgdG8gdGhlIERldmljZSBhbmQgbWF5IGhhdmUgZGlmZmVyZW50IG1lYW5pbmdzIGFtb25nIGltcGxlbWVudGF0aW9ucy5dXT48L0Rlc2NyaXB0aW9uPgogICAgICAgICAgICA8L0l0ZW0+CiAgICAgICAgICAgIDxJdGVtIElEPSIxMiI+CiAgICAgICAgICAgICAgICA8TmFtZT5SZXNldCBFcnJvciBDb2RlPC9OYW1lPgogICAgICAgICAgICAgICAgPE9wZXJhdGlvbnM+RTwvT3BlcmF0aW9ucz4KICAgICAgICAgICAgICAgIDxNdWx0aXBsZUluc3RhbmNlcz5TaW5nbGU8L011bHRpcGxlSW5zdGFuY2VzPgogICAgICAgICAgICAgICAgPE1hbmRhdG9yeT5PcHRpb25hbDwvTWFuZGF0b3J5PgogICAgICAgICAgICAgICAgPFR5cGU+PC9UeXBlPgogICAgICAgICAgICAgICAgPFJhbmdlRW51bWVyYXRpb24+PC9SYW5nZUVudW1lcmF0aW9uPgogICAgICAgICAgICAgICAgPFVuaXRzPjwvVW5pdHM+CiAgICAgICAgICAgICAgICA8RGVzY3JpcHRpb24+PCFbQ0RBVEFbRGVsZXRlIGFsbCBlcnJvciBjb2RlIFJlc291cmNlIEluc3RhbmNlcyBhbmQgY3JlYXRlIG9ubHkgb25lIHplcm8tdmFsdWUgZXJyb3IgY29kZSB0aGF0IGltcGxpZXMgbm8gZXJyb3IsIHRoZW4gcmUtZXZhbHVhdGUgYWxsIGVycm9yIGNvbmRpdGlvbnMgYW5kIHVwZGF0ZSBhbmQgY3JlYXRlIFJlc291cmNlcyBJbnN0YW5jZXMgdG8gY2FwdHVyZSBhbGwgY3VycmVudCBlcnJvciBjb25kaXRpb25zLl1dPjwvRGVzY3JpcHRpb24+CiAgICAgICAgICAgIDwvSXRlbT4KICAgICAgICAgICAgPEl0ZW0gSUQ9IjEzIj4KICAgICAgICAgICAgICAgIDxOYW1lPkN1cnJlbnQgVGltZTwvTmFtZT4KICAgICAgICAgICAgICAgIDxPcGVyYXRpb25zPlJXPC9PcGVyYXRpb25zPgogICAgICAgICAgICAgICAgPE11bHRpcGxlSW5zdGFuY2VzPlNpbmdsZTwvTXVsdGlwbGVJbnN0YW5jZXM+CiAgICAgICAgICAgICAgICA8TWFuZGF0b3J5Pk9wdGlvbmFsPC9NYW5kYXRvcnk+CiAgICAgICAgICAgICAgICA8VHlwZT5UaW1lPC9UeXBlPgogICAgICAgICAgICAgICAgPFJhbmdlRW51bWVyYXRpb24+PC9SYW5nZUVudW1lcmF0aW9uPgogICAgICAgICAgICAgICAgPFVuaXRzPjwvVW5pdHM+CiAgICAgICAgICAgICAgICA8RGVzY3JpcHRpb24+PCFbQ0RBVEFbQ3VycmVudCBVTklYIHRpbWUgb2YgdGhlIEx3TTJNIENsaWVudC4KVGhlIEx3TTJNIENsaWVudCBzaG91bGQgYmUgcmVzcG9uc2libGUgdG8gaW5jcmVhc2UgdGhpcyB0aW1lIHZhbHVlIGFzIGV2ZXJ5IHNlY29uZCBlbGFwc2VzLgpUaGUgTHdNMk0gU2VydmVyIGlzIGFibGUgdG8gd3JpdGUgdGhpcyBSZXNvdXJjZSB0byBtYWtlIHRoZSBMd00yTSBDbGllbnQgc3luY2hyb25pemVkIHdpdGggdGhlIEx3TTJNIFNlcnZlci5dXT48L0Rlc2NyaXB0aW9uPgogICAgICAgICAgICA8L0l0ZW0+CiAgICAgICAgICAgIDxJdGVtIElEPSIxNCI+CiAgICAgICAgICAgICAgICA8TmFtZT5VVEMgT2Zmc2V0PC9OYW1lPgogICAgICAgICAgICAgICAgPE9wZXJhdGlvbnM+Ulc8L09wZXJhdGlvbnM+CiAgICAgICAgICAgICAgICA8TXVsdGlwbGVJbnN0YW5jZXM+U2luZ2xlPC9NdWx0aXBsZUluc3RhbmNlcz4KICAgICAgICAgICAgICAgIDxNYW5kYXRvcnk+T3B0aW9uYWw8L01hbmRhdG9yeT4KICAgICAgICAgICAgICAgIDxUeXBlPlN0cmluZzwvVHlwZT4KICAgICAgICAgICAgICAgIDxSYW5nZUVudW1lcmF0aW9uPjwvUmFuZ2VFbnVtZXJhdGlvbj4KICAgICAgICAgICAgICAgIDxVbml0cz48L1VuaXRzPgogICAgICAgICAgICAgICAgPERlc2NyaXB0aW9uPjwhW0NEQVRBW0luZGljYXRlcyB0aGUgVVRDIG9mZnNldCBjdXJyZW50bHkgaW4gZWZmZWN0IGZvciB0aGlzIEx3TTJNIERldmljZS4gVVRDK1ggW0lTTyA4NjAxXS5dXT48L0Rlc2NyaXB0aW9uPgogICAgICAgICAgICA8L0l0ZW0+CiAgICAgICAgICAgIDxJdGVtIElEPSIxNSI+CiAgICAgICAgICAgICAgICA8TmFtZT5UaW1lem9uZTwvTmFtZT4KICAgICAgICAgICAgICAgIDxPcGVyYXRpb25zPlJXPC9PcGVyYXRpb25zPgogICAgICAgICAgICAgICAgPE11bHRpcGxlSW5zdGFuY2VzPlNpbmdsZTwvTXVsdGlwbGVJbnN0YW5jZXM+CiAgICAgICAgICAgICAgICA8TWFuZGF0b3J5Pk9wdGlvbmFsPC9NYW5kYXRvcnk+CiAgICAgICAgICAgICAgICA8VHlwZT5TdHJpbmc8L1R5cGU+CiAgICAgICAgICAgICAgICA8UmFuZ2VFbnVtZXJhdGlvbj48L1JhbmdlRW51bWVyYXRpb24+CiAgICAgICAgICAgICAgICA8VW5pdHM+PC9Vbml0cz4KICAgICAgICAgICAgICAgIDxEZXNjcmlwdGlvbj48IVtDREFUQVtJbmRpY2F0ZXMgaW4gd2hpY2ggdGltZSB6b25lIHRoZSBMd00yTSBEZXZpY2UgaXMgbG9jYXRlZCwgaW4gSUFOQSBUaW1lem9uZSAoVFopIGRhdGFiYXNlIGZvcm1hdC5dXT48L0Rlc2NyaXB0aW9uPgogICAgICAgICAgICA8L0l0ZW0+CiAgICAgICAgICAgIDxJdGVtIElEPSIxNiI+CiAgICAgICAgICAgICAgICA8TmFtZT5TdXBwb3J0ZWQgQmluZGluZyBhbmQgTW9kZXM8L05hbWU+CiAgICAgICAgICAgICAgICA8T3BlcmF0aW9ucz5SPC9PcGVyYXRpb25zPgogICAgICAgICAgICAgICAgPE11bHRpcGxlSW5zdGFuY2VzPlNpbmdsZTwvTXVsdGlwbGVJbnN0YW5jZXM+CiAgICAgICAgICAgICAgICA8TWFuZGF0b3J5Pk1hbmRhdG9yeTwvTWFuZGF0b3J5PgogICAgICAgICAgICAgICAgPFR5cGU+U3RyaW5nPC9UeXBlPgogICAgICAgICAgICAgICAgPFJhbmdlRW51bWVyYXRpb24+PC9SYW5nZUVudW1lcmF0aW9uPgogICAgICAgICAgICAgICAgPFVuaXRzPjwvVW5pdHM+CiAgICAgICAgICAgICAgICA8RGVzY3JpcHRpb24+PCFbQ0RBVEFbSW5kaWNhdGVzIHdoaWNoIGJpbmRpbmdzIGFuZCBtb2RlcyBhcmUgc3VwcG9ydGVkIGluIHRoZSBMd00yTSBDbGllbnQuIFRoZSBwb3NzaWJsZSB2YWx1ZXMgYXJlIHRob3NlIGxpc3RlZCBpbiB0aGUgTHdNMk0gQ29yZSBTcGVjaWZpY2F0aW9uLl1dPjwvRGVzY3JpcHRpb24+CiAgICAgICAgICAgIDwvSXRlbT4KICAgICAgICAgICAgPEl0ZW0gSUQ9IjE3Ij48TmFtZT5EZXZpY2UgVHlwZTwvTmFtZT4KICAgICAgICAgICAgICAgIDxPcGVyYXRpb25zPlI8L09wZXJhdGlvbnM+CiAgICAgICAgICAgICAgICA8TXVsdGlwbGVJbnN0YW5jZXM+U2luZ2xlPC9NdWx0aXBsZUluc3RhbmNlcz4KICAgICAgICAgICAgICAgIDxNYW5kYXRvcnk+T3B0aW9uYWw8L01hbmRhdG9yeT4KICAgICAgICAgICAgICAgIDxUeXBlPlN0cmluZzwvVHlwZT4KICAgICAgICAgICAgICAgIDxSYW5nZUVudW1lcmF0aW9uPjwvUmFuZ2VFbnVtZXJhdGlvbj4KICAgICAgICAgICAgICAgIDxVbml0cz48L1VuaXRzPgogICAgICAgICAgICAgICAgPERlc2NyaXB0aW9uPjwhW0NEQVRBW1R5cGUgb2YgdGhlIGRldmljZSAobWFudWZhY3R1cmVyIHNwZWNpZmllZCBzdHJpbmc6IGUuZy4gc21hcnQgbWV0ZXJzIC8gZGV2IENsYXNzIC8gLi4uKV1dPjwvRGVzY3JpcHRpb24+CiAgICAgICAgICAgIDwvSXRlbT4KICAgICAgICAgICAgPEl0ZW0gSUQ9IjE4Ij48TmFtZT5IYXJkd2FyZSBWZXJzaW9uPC9OYW1lPgogICAgICAgICAgICAgICAgPE9wZXJhdGlvbnM+UjwvT3BlcmF0aW9ucz4KICAgICAgICAgICAgICAgIDxNdWx0aXBsZUluc3RhbmNlcz5TaW5nbGU8L011bHRpcGxlSW5zdGFuY2VzPgogICAgICAgICAgICAgICAgPE1hbmRhdG9yeT5PcHRpb25hbDwvTWFuZGF0b3J5PgogICAgICAgICAgICAgICAgPFR5cGU+U3RyaW5nPC9UeXBlPgogICAgICAgICAgICAgICAgPFJhbmdlRW51bWVyYXRpb24+PC9SYW5nZUVudW1lcmF0aW9uPgogICAgICAgICAgICAgICAgPFVuaXRzPjwvVW5pdHM+CiAgICAgICAgICAgICAgICA8RGVzY3JpcHRpb24+PCFbQ0RBVEFbQ3VycmVudCBoYXJkd2FyZSB2ZXJzaW9uIG9mIHRoZSBkZXZpY2VdXT48L0Rlc2NyaXB0aW9uPgogICAgICAgICAgICA8L0l0ZW0+CiAgICAgICAgICAgIDxJdGVtIElEPSIxOSI+PE5hbWU+U29mdHdhcmUgVmVyc2lvbjwvTmFtZT4KICAgICAgICAgICAgICAgIDxPcGVyYXRpb25zPlI8L09wZXJhdGlvbnM+CiAgICAgICAgICAgICAgICA8TXVsdGlwbGVJbnN0YW5jZXM+U2luZ2xlPC9NdWx0aXBsZUluc3RhbmNlcz4KICAgICAgICAgICAgICAgIDxNYW5kYXRvcnk+T3B0aW9uYWw8L01hbmRhdG9yeT4KICAgICAgICAgICAgICAgIDxUeXBlPlN0cmluZzwvVHlwZT4KICAgICAgICAgICAgICAgIDxSYW5nZUVudW1lcmF0aW9uPjwvUmFuZ2VFbnVtZXJhdGlvbj4KICAgICAgICAgICAgICAgIDxVbml0cz48L1VuaXRzPgogICAgICAgICAgICAgICAgPERlc2NyaXB0aW9uPjwhW0NEQVRBW0N1cnJlbnQgc29mdHdhcmUgdmVyc2lvbiBvZiB0aGUgZGV2aWNlIChtYW51ZmFjdHVyZXIgc3BlY2lmaWVkIHN0cmluZykuIE9uIGVsYWJvcmF0ZWQgTHdNMk0gZGV2aWNlLCBTVyBjb3VsZCBiZSBzcGxpdCBpbiAyIHBhcnRzOiBhIGZpcm13YXJlIG9uZSBhbmQgYSBoaWdoZXIgbGV2ZWwgc29mdHdhcmUgb24gdG9wLgpCb3RoIHBpZWNlcyBvZiBTb2Z0d2FyZSBhcmUgdG9nZXRoZXIgbWFuYWdlZCBieSBMd00yTSBGaXJtd2FyZSBVcGRhdGUgT2JqZWN0IChPYmplY3QgSUQgNSldXT48L0Rlc2NyaXB0aW9uPgogICAgICAgICAgICA8L0l0ZW0+CiAgICAgICAgICAgIDxJdGVtIElEPSIyMCI+PE5hbWU+QmF0dGVyeSBTdGF0dXM8L05hbWU+CiAgICAgICAgICAgICAgICA8T3BlcmF0aW9ucz5SPC9PcGVyYXRpb25zPgogICAgICAgICAgICAgICAgPE11bHRpcGxlSW5zdGFuY2VzPlNpbmdsZTwvTXVsdGlwbGVJbnN0YW5jZXM+CiAgICAgICAgICAgICAgICA8TWFuZGF0b3J5Pk9wdGlvbmFsPC9NYW5kYXRvcnk+CiAgICAgICAgICAgICAgICA8VHlwZT5JbnRlZ2VyPC9UeXBlPgogICAgICAgICAgICAgICAgPFJhbmdlRW51bWVyYXRpb24+MC4uNjwvUmFuZ2VFbnVtZXJhdGlvbj4KICAgICAgICAgICAgICAgIDxVbml0cz48L1VuaXRzPgogICAgICAgICAgICAgICAgPERlc2NyaXB0aW9uPjwhW0NEQVRBW1RoaXMgdmFsdWUgaXMgb25seSB2YWxpZCBmb3IgdGhlIERldmljZSBJbnRlcm5hbCBCYXR0ZXJ5IGlmIHByZXNlbnQgKG9uZSBBdmFpbGFibGUgUG93ZXIgU291cmNlcyBSZXNvdXJjZSBJbnN0YW5jZSB2YWx1ZSBpcyAxKS4KQmF0dGVyeQpTdGF0dXMJTWVhbmluZwlEZXNjcmlwdGlvbgowCU5vcm1hbAlUaGUgYmF0dGVyeSBpcyBvcGVyYXRpbmcgbm9ybWFsbHkgYW5kIG5vdCBvbiBwb3dlci4KMQlDaGFyZ2luZwlUaGUgYmF0dGVyeSBpcyBjdXJyZW50bHkgY2hhcmdpbmcuCjIJQ2hhcmdlIENvbXBsZXRlCVRoZSBiYXR0ZXJ5IGlzIGZ1bGx5IGNoYXJnZWQgYW5kIHN0aWxsIG9uIHBvd2VyLgozCURhbWFnZWQJVGhlIGJhdHRlcnkgaGFzIHNvbWUgcHJvYmxlbS4KNAlMb3cgQmF0dGVyeQlUaGUgYmF0dGVyeSBpcyBsb3cgb24gY2hhcmdlLgo1CU5vdCBJbnN0YWxsZWQJVGhlIGJhdHRlcnkgaXMgbm90IGluc3RhbGxlZC4KNglVbmtub3duCVRoZSBiYXR0ZXJ5IGluZm9ybWF0aW9uIGlzIG5vdCBhdmFpbGFibGUuXV0+PC9EZXNjcmlwdGlvbj4KICAgICAgICAgICAgPC9JdGVtPgogICAgICAgICAgICA8SXRlbSBJRD0iMjEiPjxOYW1lPk1lbW9yeSBUb3RhbDwvTmFtZT4KICAgICAgICAgICAgICAgIDxPcGVyYXRpb25zPlI8L09wZXJhdGlvbnM+CiAgICAgICAgICAgICAgICA8TXVsdGlwbGVJbnN0YW5jZXM+U2luZ2xlPC9NdWx0aXBsZUluc3RhbmNlcz4KICAgICAgICAgICAgICAgIDxNYW5kYXRvcnk+T3B0aW9uYWw8L01hbmRhdG9yeT4KICAgICAgICAgICAgICAgIDxUeXBlPkludGVnZXI8L1R5cGU+CiAgICAgICAgICAgICAgICA8UmFuZ2VFbnVtZXJhdGlvbj48L1JhbmdlRW51bWVyYXRpb24+CiAgICAgICAgICAgICAgICA8VW5pdHM+PC9Vbml0cz4KICAgICAgICAgICAgICAgIDxEZXNjcmlwdGlvbj48IVtDREFUQVtUb3RhbCBhbW91bnQgb2Ygc3RvcmFnZSBzcGFjZSB3aGljaCBjYW4gc3RvcmUgZGF0YSBhbmQgc29mdHdhcmUgaW4gdGhlIEx3TTJNIERldmljZSAoZXhwcmVzc2VkIGluIGtpbG9ieXRlcykuICBOb3RlOiAxIGtpbG9ieXRlIGNvcnJlc3BvbmRzIHRvIDEwMDAgYnl0ZXMuXV0+PC9EZXNjcmlwdGlvbj4KICAgICAgICAgICAgPC9JdGVtPgogICAgICAgICAgICA8SXRlbSBJRD0iMjIiPjxOYW1lPkV4dERldkluZm88L05hbWU+CiAgICAgICAgICAgICAgICA8T3BlcmF0aW9ucz5SPC9PcGVyYXRpb25zPgogICAgICAgICAgICAgICAgPE11bHRpcGxlSW5zdGFuY2VzPk11bHRpcGxlPC9NdWx0aXBsZUluc3RhbmNlcz4KICAgICAgICAgICAgICAgIDxNYW5kYXRvcnk+T3B0aW9uYWw8L01hbmRhdG9yeT4KICAgICAgICAgICAgICAgIDxUeXBlPk9iamxuazwvVHlwZT4KICAgICAgICAgICAgICAgIDxSYW5nZUVudW1lcmF0aW9uPjwvUmFuZ2VFbnVtZXJhdGlvbj4KICAgICAgICAgICAgICAgIDxVbml0cz48L1VuaXRzPgogICAgICAgICAgICAgICAgPERlc2NyaXB0aW9uPjwhW0NEQVRBW1JlZmVyZW5jZSB0byBleHRlcm5hbCAiRGV2aWNlIiBvYmplY3QgaW5zdGFuY2UgY29udGFpbmluZyBpbmZvcm1hdGlvbi4gRm9yIGV4YW1wbGUsIHN1Y2ggYW4gZXh0ZXJuYWwgZGV2aWNlIGNhbiBiZSBhIEhvc3QgRGV2aWNlLCB3aGljaCBpcyBhIGRldmljZSBpbnRvIHdoaWNoIHRoZSBEZXZpY2UgY29udGFpbmluZyB0aGUgTHdNMk0gY2xpZW50IGlzIGVtYmVkZGVkLiBUaGlzIFJlc291cmNlIG1heSBiZSB1c2VkIHRvIHJldHJpZXZlIGluZm9ybWF0aW9uIGFib3V0IHRoZSBIb3N0IERldmljZS5dXT48L0Rlc2NyaXB0aW9uPgogICAgICAgICAgICA8L0l0ZW0+PC9SZXNvdXJjZXM+CiAgICAgICAgPERlc2NyaXB0aW9uMj48L0Rlc2NyaXB0aW9uMj4KICAgIDwvT2JqZWN0Pgo8L0xXTTJNPgo=" } \ No newline at end of file diff --git a/monitoring/src/main/resources/root_rule_chain.json b/monitoring/src/main/resources/root_rule_chain.json index 46bdc72d9f..a1c12c8e9d 100644 --- a/monitoring/src/main/resources/root_rule_chain.json +++ b/monitoring/src/main/resources/root_rule_chain.json @@ -39,8 +39,11 @@ "type": "org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode", "name": "Save Attributes", "singletonMode": false, - "configurationVersion": 1, + "configurationVersion": 3, "configuration": { + "processingSettings": { + "type": "ON_EVERY_MESSAGE" + }, "scope": "CLIENT_SCOPE", "notifyDevice": false, "sendAttributesUpdatedNotification": false, diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 77567163c6..65def6a964 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -51,6 +51,7 @@ public class ContainerTestSuite { private static final String TB_CORE_LOG_REGEXP = ".*Starting polling for events.*"; private static final String TRANSPORTS_LOG_REGEXP = ".*Going to recalculate partitions.*"; private static final String TB_VC_LOG_REGEXP = TRANSPORTS_LOG_REGEXP; + private static final String TB_EDQS_LOG_REGEXP = ".*All partitions processed.*"; private static final String TB_JS_EXECUTOR_LOG_REGEXP = ".*template started.*"; private static final Duration CONTAINER_STARTUP_TIMEOUT = Duration.ofSeconds(400); @@ -114,6 +115,8 @@ public class ContainerTestSuite { List composeFiles = new ArrayList<>(Arrays.asList( new File(targetDir + "docker-compose.yml"), + new File(targetDir + "docker-compose.edqs.yml"), + new File(targetDir + "docker-compose.edqs.volumes.yml"), new File(targetDir + "docker-compose.volumes.yml"), new File(targetDir + "docker-compose.mosquitto.yml"), new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid.yml" : "docker-compose.postgres.yml")), @@ -174,13 +177,19 @@ public class ContainerTestSuite { .withExposedService("broker", 1883) .waitingFor("tb-core1", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-core2", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-rule-engine1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-rule-engine2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-http-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-http-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-mqtt-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-mqtt-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-coap-transport", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-lwm2m-transport", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-vc-executor1", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-js-executor", Wait.forLogMessage(TB_JS_EXECUTOR_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); + .waitingFor("tb-js-executor", Wait.forLogMessage(TB_JS_EXECUTOR_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-edqs-1", Wait.forLogMessage(TB_EDQS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-edqs-2", Wait.forLogMessage(TB_EDQS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); testContainer.start(); setActive(true); } catch (Exception e) { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java index 94d0c64fa0..d74438a428 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java @@ -35,6 +35,7 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; @@ -56,6 +57,9 @@ import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rpc.Rpc; @@ -66,7 +70,6 @@ import org.thingsboard.server.common.data.security.DeviceCredentials; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.UUID; import static io.restassured.RestAssured.given; import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; @@ -110,6 +113,20 @@ public class TestRestClient { requestSpec.header(JWT_TOKEN_HEADER_PARAM, "Bearer " + token); } + public void resetToken() { + token = null; + refreshToken = null; + } + + public Tenant postTenant(Tenant tenant) { + return given().spec(requestSpec).body(tenant) + .post("/api/tenant") + .then() + .statusCode(HTTP_OK) + .extract() + .as(Tenant.class); + } + public Device postDevice(String accessToken, Device device) { return given().spec(requestSpec).body(device) .pathParams("accessToken", accessToken) @@ -220,6 +237,15 @@ public class TestRestClient { .as(JsonNode.class); } + public JsonNode getLatestTelemetry(EntityId entityId) { + return given().spec(requestSpec) + .get("/api/plugins/telemetry/" + entityId.getEntityType().name() + "/" + entityId.getId() + "/values/timeseries") + .then() + .statusCode(HTTP_OK) + .extract() + .as(JsonNode.class); + } + public JsonPath postProvisionRequest(String provisionRequest) { return given().spec(requestSpec) .body(provisionRequest) @@ -479,6 +505,28 @@ public class TestRestClient { .as(User.class); } + public UserId createUserAndLogin(User user, String password) { + UserId userId = postUser(user).getId(); + getAndSetUserToken(userId); + return userId; + } + + public void getAndSetUserToken(UserId id) { + ObjectNode tokenInfo = given().spec(requestSpec) + .get("/api/user/" + id.getId().toString() + "/token") + .then() + .extract() + .as(ObjectNode.class); + token = tokenInfo.get("token").asText(); + refreshToken = tokenInfo.get("refreshToken").asText(); + requestSpec.header(JWT_TOKEN_HEADER_PARAM, "Bearer " + token); + } + + protected void resetTokens() { + this.token = null; + this.refreshToken = null; + } + public void deleteUser(UserId userId) { given().spec(requestSpec) .delete("/api/user/{userId}", userId.getId()) @@ -643,4 +691,45 @@ public class TestRestClient { } return urlParams; } + + public PageData postEntityDataQuery(EntityDataQuery entityDataQuery) { + return given().spec(requestSpec).body(entityDataQuery) + .post("/api/entitiesQuery/find") + .then() + .statusCode(HTTP_OK) + .extract() + .as(new TypeRef<>() {}); + } + + public Long postCountDataQuery(EntityCountQuery entityCountQuery) { + return given().spec(requestSpec).body(entityCountQuery) + .post("/api/entitiesQuery/count") + .then() + .statusCode(HTTP_OK) + .extract() + .as(Long.class); + } + + public Boolean isEdqsApiEnabled() { + return given().spec(requestSpec) + .get("/api/edqs/enabled") + .then() + .statusCode(HTTP_OK) + .extract() + .as(Boolean.class); + } + + public void assignDeviceToCustomer(CustomerId customerId, DeviceId id) { + given().spec(requestSpec) + .post("/api/customer/" + customerId.getId().toString() + "/device/" + id.getId().toString()) + .then() + .statusCode(HTTP_OK); + } + + public void deleteTenant(TenantId tenantId) { + given().spec(requestSpec) + .delete("/api/tenant/" + tenantId.getId().toString()) + .then() + .statusCode(HTTP_OK); + } } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index a71abe1781..9c1a90a4f1 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -48,6 +48,7 @@ public class ThingsBoardDbInstaller { private final static String TB_MQTT_TRANSPORT_LOG_VOLUME = "tb-mqtt-transport-log-test-volume"; private final static String TB_SNMP_TRANSPORT_LOG_VOLUME = "tb-snmp-transport-log-test-volume"; private final static String TB_VC_EXECUTOR_LOG_VOLUME = "tb-vc-executor-log-test-volume"; + private final static String TB_EDQS_LOG_VOLUME = "tb-edqs-log-test-volume"; private final static String JAVA_OPTS = "-Xmx512m"; private final DockerComposeExecutor dockerCompose; @@ -65,6 +66,7 @@ public class ThingsBoardDbInstaller { private final String tbMqttTransportLogVolume; private final String tbSnmpTransportLogVolume; private final String tbVcExecutorLogVolume; + private final String tbEdqsLogVolume; private final Map env; public ThingsBoardDbInstaller() { @@ -103,6 +105,7 @@ public class ThingsBoardDbInstaller { tbMqttTransportLogVolume = project + "_" + TB_MQTT_TRANSPORT_LOG_VOLUME; tbSnmpTransportLogVolume = project + "_" + TB_SNMP_TRANSPORT_LOG_VOLUME; tbVcExecutorLogVolume = project + "_" + TB_VC_EXECUTOR_LOG_VOLUME; + tbEdqsLogVolume = project + "_" + TB_EDQS_LOG_VOLUME; dockerCompose = new DockerComposeExecutor(composeFiles, project); @@ -119,6 +122,7 @@ public class ThingsBoardDbInstaller { env.put("TB_MQTT_TRANSPORT_LOG_VOLUME", tbMqttTransportLogVolume); env.put("TB_SNMP_TRANSPORT_LOG_VOLUME", tbSnmpTransportLogVolume); env.put("TB_VC_EXECUTOR_LOG_VOLUME", tbVcExecutorLogVolume); + env.put("TB_EDQS_LOG_VOLUME", tbEdqsLogVolume); if (IS_REDIS_CLUSTER) { for (int i = 0; i < 6; i++) { env.put("REDIS_CLUSTER_DATA_VOLUME_" + i, redisClusterDataVolume + '-' + i); @@ -189,6 +193,9 @@ public class ThingsBoardDbInstaller { dockerCompose.withCommand("volume create " + tbVcExecutorLogVolume); dockerCompose.invokeDocker(); + dockerCompose.withCommand("volume create " + tbEdqsLogVolume); + dockerCompose.invokeDocker(); + StringBuilder additionalServices = new StringBuilder(); if (IS_HYBRID_MODE) { additionalServices.append(" cassandra"); @@ -220,7 +227,8 @@ public class ThingsBoardDbInstaller { dockerCompose.withCommand("up -d postgres" + additionalServices); dockerCompose.invokeCompose(); - dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb-core1"); + dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true " + + "tb-core1"); dockerCompose.invokeCompose(); } finally { @@ -240,6 +248,7 @@ public class ThingsBoardDbInstaller { copyLogs(tbMqttTransportLogVolume, "./target/tb-mqtt-transport-logs/"); copyLogs(tbSnmpTransportLogVolume, "./target/tb-snmp-transport-logs/"); copyLogs(tbVcExecutorLogVolume, "./target/tb-vc-executor-logs/"); + copyLogs(tbEdqsLogVolume, "./target/tb-edqs-logs/"); StringJoiner rmVolumesCommand = new StringJoiner(" ") .add("volume rm -f") @@ -251,6 +260,7 @@ public class ThingsBoardDbInstaller { .add(tbMqttTransportLogVolume) .add(tbSnmpTransportLogVolume) .add(tbVcExecutorLogVolume) + .add(tbEdqsLogVolume) .add(resolveRedisComposeVolumeLog()); if (IS_HYBRID_MODE) { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java new file mode 100644 index 0000000000..53d8a72e7f --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java @@ -0,0 +1,214 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.msa.edqs; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.DeviceTypeFilter; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.common.data.query.TsValue; +import org.thingsboard.server.msa.AbstractContainerTest; +import org.thingsboard.server.msa.DisableUIListeners; +import org.thingsboard.server.msa.ui.utils.EntityPrototypes; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultCustomer; +import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultCustomerAdmin; +import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultDeviceProfile; +import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultTenantAdmin; + +@DisableUIListeners +public class EdqsEntityDataQueryTest extends AbstractContainerTest { + + private TenantId tenantId; + private CustomerId customerId; + private TenantId tenantId2; + private CustomerId customerId2; + private UserId tenantAdminId; + private UserId customerUserId; + private UserId tenant2AdminId; + private UserId customer2UserId; + private final List tenantDevices = new ArrayList<>(); + private final List tenant2Devices = new ArrayList<>(); + private final String deviceProfile = "LoRa-" + RandomStringUtils.randomAlphabetic(10); + + @BeforeClass + public void beforeClass() throws Exception { + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + await().atMost(60, TimeUnit.SECONDS).until(() -> testRestClient.isEdqsApiEnabled()); + + tenantId = testRestClient.postTenant(EntityPrototypes.defaultTenantPrototype("Tenant")).getId(); + tenantAdminId = testRestClient.createUserAndLogin(defaultTenantAdmin(tenantId, "tenantAdmin@thingsboard.org"), "tenant"); + testRestClient.postDeviceProfile(defaultDeviceProfile(deviceProfile)); + createDevices(deviceProfile, tenantDevices, 97); + customerId = testRestClient.postCustomer(defaultCustomer(tenantId, "Customer")).getId(); + customerUserId = testRestClient.postUser(defaultCustomerAdmin(tenantId, customerId, "customerUser@thingsboard.org")).getId(); + assignDevicesToCustomer(customerId, tenantDevices, 12); + + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + tenantId2 = testRestClient.postTenant(EntityPrototypes.defaultTenantPrototype("Tenant")).getId(); + tenant2AdminId = testRestClient.createUserAndLogin(defaultTenantAdmin(tenantId2, "tenant2Admin@thingsboard.org"), "tenant"); + testRestClient.postDeviceProfile(defaultDeviceProfile(deviceProfile)); + createDevices(deviceProfile, tenant2Devices, 97); + customerId2 = testRestClient.postCustomer(defaultCustomer(tenantId2, "Customer")).getId(); + customer2UserId = testRestClient.postUser(defaultCustomerAdmin(tenantId2, customerId2, "customer2User@thingsboard.org")).getId(); + assignDevicesToCustomer(customerId2, tenant2Devices, 12); + } + + @BeforeMethod + public void beforeMethod() { + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + } + + @AfterClass + public void afterClass() { + testRestClient.resetToken(); + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + testRestClient.deleteTenant(tenantId); + testRestClient.deleteTenant(tenantId2); + } + + @Test + public void testSysAdminCountEntitiesByQuery() { + EntityTypeFilter allDeviceFilter = new EntityTypeFilter(); + allDeviceFilter.setEntityType(EntityType.DEVICE); + EntityCountQuery query = new EntityCountQuery(allDeviceFilter); + await("Waiting for total device count") + .atMost(30, TimeUnit.SECONDS) + .until(() -> testRestClient.postCountDataQuery(query).compareTo(97L * 2) >= 0); + + testRestClient.getAndSetUserToken(tenantAdminId); + await("Waiting for total device count") + .atMost(30, TimeUnit.SECONDS) + .until(() -> testRestClient.postCountDataQuery(query).equals(97L)); + + testRestClient.resetToken(); + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + testRestClient.getAndSetUserToken(tenant2AdminId); + await("Waiting for total device count") + .atMost(30, TimeUnit.SECONDS) + .until(() -> testRestClient.postCountDataQuery(query).equals(97L)); + } + + @Test + public void testRetrieveTenantDevicesByDeviceTypeFilter() { + // login tenant admin + testRestClient.getAndSetUserToken(tenantAdminId); + checkUserDevices(tenantDevices); + + // login customer user + testRestClient.getAndSetUserToken(customerUserId); + checkUserDevices(tenantDevices.subList(0, 12)); + + // login other tenant admin + testRestClient.resetToken(); + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + testRestClient.getAndSetUserToken(tenant2AdminId); + checkUserDevices(tenant2Devices); + } + + private void checkUserDevices(List devices) { + DeviceTypeFilter filter = new DeviceTypeFilter(); + filter.setDeviceTypes(List.of(deviceProfile)); + filter.setDeviceNameFilter(""); + + EntityDataSortOrder sortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder); + List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + List latestFields = Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestFields, null); + + EntityTypeFilter allDeviceFilter = new EntityTypeFilter(); + allDeviceFilter.setEntityType(EntityType.DEVICE); + EntityCountQuery countQuery = new EntityCountQuery(allDeviceFilter); + await("Waiting for total device count") + .atMost(30, TimeUnit.SECONDS) + .until(() -> testRestClient.postCountDataQuery(countQuery).intValue() == devices.size()); + + PageData result = testRestClient.postEntityDataQuery(query); + assertThat(result.getTotalElements()).isEqualTo(devices.size()); + List retrievedDevices = result.getData(); + + assertThat(retrievedDevices).hasSize(10); + List retrievedDeviceNames = retrievedDevices.stream().map(entityData -> entityData.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()).toList(); + assertThat(retrievedDeviceNames).containsExactlyInAnyOrderElementsOf(devices.stream().map(Device::getName).toList().subList(0, 10)); + + //check temperature + for (int i = 0; i < 10; i++) { + Map> latest = retrievedDevices.get(i).getLatest(); + String name = latest.get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(latest.get(EntityKeyType.TIME_SERIES).get("temperature").getValue()).isEqualTo(name.substring(name.length() - 1)); + } + } + + private String createDevices(String deviceType, List tenantDevices, int deviceCount) throws InterruptedException { + String prefix = StringUtils.randomAlphabetic(5); + for (int i = 0; i < deviceCount; i++) { + Device device = new Device(); + device.setName(prefix + "Device" + i); + device.setType(deviceType); + device.setLabel("testLabel" + (int) (Math.random() * 1000)); + //TO make sure devices have different created time + Thread.sleep(1); + String token = RandomStringUtils.randomAlphabetic(10); + Device saved = testRestClient.postDevice(token, device); + tenantDevices.add(saved); + + // save timeseries data + testRestClient.postTelemetry(token, createDeviceTelemetry(i)); + } + return deviceType; + } + + private void assignDevicesToCustomer(CustomerId customerId, List devices, int deviceCount) { + for (int i = 0; i < deviceCount; i++) { + Device device = devices.get(i); + testRestClient.assignDeviceToCustomer(customerId, device.getId()); + } + } + + protected ObjectNode createDeviceTelemetry(int temperature) { + ObjectNode objectNode = mapper.createObjectNode(); + objectNode.put("temperature", temperature); + return objectNode; + } + +} diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/utils/EntityPrototypes.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/utils/EntityPrototypes.java index ce5fee1f6f..91620271b7 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/utils/EntityPrototypes.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/utils/EntityPrototypes.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.DeviceProfileProvisionType; import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmSeverity; @@ -37,12 +38,26 @@ import org.thingsboard.server.common.data.device.profile.DisabledDeviceProfilePr import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.security.Authority; public class EntityPrototypes { + public static Tenant defaultTenantPrototype(String tenantName) { + Tenant tenant = new Tenant(); + tenant.setTitle(tenantName); + return tenant; + } + + public static Customer defaultCustomer(TenantId tenantId, String title) { + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle(title); + return customer; + } + public static Customer defaultCustomerPrototype(String entityName) { Customer customer = new Customer(); customer.setTitle(entityName); @@ -169,6 +184,23 @@ public class EntityPrototypes { return user; } + public static User defaultTenantAdmin(TenantId tenantId, String email) { + User user = new User(); + user.setTenantId(tenantId); + user.setEmail(email); + user.setAuthority(Authority.TENANT_ADMIN); + return user; + } + + public static User defaultCustomerAdmin(TenantId tenantId, CustomerId customerId, String email) { + User user = new User(); + user.setTenantId(tenantId); + user.setCustomerId(customerId); + user.setEmail(email); + user.setAuthority(Authority.CUSTOMER_USER); + return user; + } + public static User defaultUser(String email, CustomerId customerId, String name) { User user = new User(); user.setEmail(email); diff --git a/msa/black-box-tests/src/test/resources/connectivity.xml b/msa/black-box-tests/src/test/resources/connectivity.xml index 2bde3f0a3f..425fbd67eb 100644 --- a/msa/black-box-tests/src/test/resources/connectivity.xml +++ b/msa/black-box-tests/src/test/resources/connectivity.xml @@ -22,6 +22,7 @@ + \ No newline at end of file diff --git a/msa/edqs/docker/Dockerfile b/msa/edqs/docker/Dockerfile new file mode 100644 index 0000000000..e9099c09c5 --- /dev/null +++ b/msa/edqs/docker/Dockerfile @@ -0,0 +1,31 @@ +# +# Copyright © 2016-2025 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM thingsboard/openjdk17:bookworm-slim + +COPY start-tb-edqs.sh ${pkg.name}.deb /tmp/ + +RUN chmod a+x /tmp/*.sh \ + && mv /tmp/start-tb-edqs.sh /usr/bin && \ + (yes | dpkg -i /tmp/${pkg.name}.deb) && \ + rm /tmp/${pkg.name}.deb && \ + (systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || :) && \ + chown -R ${pkg.user}:${pkg.user} /tmp && \ + chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar + +USER ${pkg.user} + +CMD ["start-tb-edqs.sh"] \ No newline at end of file diff --git a/msa/edqs/docker/start-tb-edqs.sh b/msa/edqs/docker/start-tb-edqs.sh new file mode 100755 index 0000000000..deb0f70eff --- /dev/null +++ b/msa/edqs/docker/start-tb-edqs.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# Copyright © 2016-2025 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +CONF_FOLDER=${pkg.installFolder}/conf +jarfile=${pkg.installFolder}/bin/${pkg.name}.jar +configfile=${pkg.name}.conf + +source "${CONF_FOLDER}/${configfile}" + +echo "Starting '${project.name}' ..." + +cd ${pkg.installFolder}/bin + +exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.edqs.ThingsboardEdqsApplication \ + -Dspring.jpa.hibernate.ddl-auto=none \ + -Dlogging.config=$CONF_FOLDER/logback.xml \ + org.springframework.boot.loader.launch.PropertiesLauncher diff --git a/msa/edqs/pom.xml b/msa/edqs/pom.xml new file mode 100644 index 0000000000..f22cb0187c --- /dev/null +++ b/msa/edqs/pom.xml @@ -0,0 +1,190 @@ + + + 4.0.0 + + org.thingsboard + 4.0.0-SNAPSHOT + msa + + org.thingsboard.msa + edqs + pom + + ThingsBoard Entity Data Query Microservice + https://thingsboard.io + ThingsBoard Entity Data Query Microservice + + + UTF-8 + ${basedir}/../.. + edqs + tb-edqs + /var/log/${pkg.name} + /usr/share/${pkg.name} + pre-integration-test + + + + + org.thingsboard + edqs + ${project.version} + deb + deb + provided + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-edqs + package + + copy + + + + + org.thingsboard + edqs + deb + deb + ${pkg.name}.deb + ${project.build.directory} + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + copy-docker-config + process-resources + + copy-resources + + + ${project.build.directory} + + + docker + true + + + + + + + + com.spotify + dockerfile-maven-plugin + + + build-docker-image + pre-integration-test + + build + + + ${dockerfile.skip} + ${docker.repo}/${docker.name} + true + false + ${project.build.directory} + + + + tag-docker-image + pre-integration-test + + tag + + + ${dockerfile.skip} + ${docker.repo}/${docker.name} + ${project.version} + + + + + + + + + push-docker-image + + + push-docker-image + + + + + + com.spotify + dockerfile-maven-plugin + + + push-latest-docker-image + pre-integration-test + + push + + + latest + ${docker.repo}/${docker.name} + + + + push-version-docker-image + pre-integration-test + + push + + + ${project.version} + ${docker.repo}/${docker.name} + + + + + + + + + + + jenkins + Jenkins Repository + https://repo.jenkins-ci.org/releases + + false + + + + diff --git a/msa/pom.xml b/msa/pom.xml index 5ae0e903c1..98e820daaf 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -48,6 +48,7 @@ transport js-executor monitoring + edqs diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 1d4d1592cf..f0b1426cc7 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -151,6 +151,8 @@ queue: core: # Default topic name topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + # For high-priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_CORE_NOTIFICATIONS_TOPIC:tb_core.notifications}" # Interval in milliseconds to poll messages by Core microservices poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" # Amount of partitions used by Core microservices diff --git a/pom.xml b/pom.xml index b851b8fc10..c0dc7384f7 100755 --- a/pom.xml +++ b/pom.xml @@ -74,6 +74,7 @@ 1.7.0 4.4.0 2.2.14 + 0.6.12 3.12.1 2.0.0-M15 2.10.1 @@ -83,7 +84,7 @@ 3.9.2 3.25.5 1.63.0 - 1.2.5 + 1.2.6 1.18.32 1.2.5 1.2.5 @@ -165,6 +166,8 @@ 1.6.1 2.19.0 9.2.0 + 1.1.10.5 + 9.10.0 @@ -172,6 +175,7 @@ common rule-engine dao + edqs transport ui-ngx tools @@ -848,6 +852,8 @@ .run/** **/NetworkReceive.java **/lwm2m-registry/** + **/test/resources/lwm2m/** + **/resources/lwm2m/models/** src/main/data/resources/** @@ -1031,6 +1037,11 @@ coap-server ${project.version} + + org.thingsboard.common + edqs + ${project.version} + org.thingsboard.common.script script-api @@ -2279,6 +2290,16 @@ metadata-extractor ${drewnoakes-metadata-extractor.version} + + org.xerial.snappy + snappy-java + ${snappy.version} + + + org.rocksdb + rocksdbjni + ${rocksdbjni.version} + diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesDeleteRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesDeleteRequest.java index f1142443ea..374fcc45f6 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesDeleteRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesDeleteRequest.java @@ -21,22 +21,31 @@ import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.ToString; +import org.thingsboard.common.util.NoOpFutureCallback; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; import java.util.List; +import java.util.UUID; + +import static java.util.Objects.requireNonNullElse; @Getter @ToString @AllArgsConstructor(access = AccessLevel.PRIVATE) -public class AttributesDeleteRequest { +public class AttributesDeleteRequest implements CalculatedFieldSystemAwareRequest { private final TenantId tenantId; private final EntityId entityId; private final AttributeScope scope; private final List keys; private final boolean notifyDevice; + private final List previousCalculatedFieldIds; + private final UUID tbMsgId; + private final TbMsgType tbMsgType; private final FutureCallback callback; public static Builder builder() { @@ -50,6 +59,9 @@ public class AttributesDeleteRequest { private AttributeScope scope; private List keys; private boolean notifyDevice; + private List previousCalculatedFieldIds; + private UUID tbMsgId; + private TbMsgType tbMsgType; private FutureCallback callback; Builder() {} @@ -89,6 +101,21 @@ public class AttributesDeleteRequest { return this; } + public Builder previousCalculatedFieldIds(List previousCalculatedFieldIds) { + this.previousCalculatedFieldIds = previousCalculatedFieldIds; + return this; + } + + public Builder tbMsgId(UUID tbMsgId) { + this.tbMsgId = tbMsgId; + return this; + } + + public Builder tbMsgType(TbMsgType tbMsgType) { + this.tbMsgType = tbMsgType; + return this; + } + public Builder callback(FutureCallback callback) { this.callback = callback; return this; @@ -109,7 +136,9 @@ public class AttributesDeleteRequest { } public AttributesDeleteRequest build() { - return new AttributesDeleteRequest(tenantId, entityId, scope, keys, notifyDevice, callback); + return new AttributesDeleteRequest( + tenantId, entityId, scope, keys, notifyDevice, previousCalculatedFieldIds, tbMsgId, tbMsgType, requireNonNullElse(callback, NoOpFutureCallback.instance()) + ); } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java index c7d09d1525..c3095836bd 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java @@ -21,27 +21,45 @@ import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.ToString; +import org.thingsboard.common.util.NoOpFutureCallback; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; import java.util.List; +import java.util.UUID; + +import static java.util.Objects.requireNonNullElse; @Getter @ToString @AllArgsConstructor(access = AccessLevel.PRIVATE) -public class AttributesSaveRequest { +public class AttributesSaveRequest implements CalculatedFieldSystemAwareRequest { private final TenantId tenantId; private final EntityId entityId; private final AttributeScope scope; private final List entries; private final boolean notifyDevice; + private final Strategy strategy; + private final List previousCalculatedFieldIds; + private final UUID tbMsgId; + private final TbMsgType tbMsgType; private final FutureCallback callback; + public record Strategy(boolean saveAttributes, boolean sendWsUpdate, boolean processCalculatedFields) { + + public static final Strategy PROCESS_ALL = new Strategy(true, true, true); + public static final Strategy WS_ONLY = new Strategy(false, true, false); + public static final Strategy SKIP_ALL = new Strategy(false, false, false); + + } + public static Builder builder() { return new Builder(); } @@ -53,6 +71,10 @@ public class AttributesSaveRequest { private AttributeScope scope; private List entries; private boolean notifyDevice = true; + private Strategy strategy; + private List previousCalculatedFieldIds; + private UUID tbMsgId; + private TbMsgType tbMsgType; private FutureCallback callback; Builder() {} @@ -100,6 +122,26 @@ public class AttributesSaveRequest { return this; } + public Builder strategy(Strategy strategy) { + this.strategy = strategy; + return this; + } + + public Builder previousCalculatedFieldIds(List previousCalculatedFieldIds) { + this.previousCalculatedFieldIds = previousCalculatedFieldIds; + return this; + } + + public Builder tbMsgId(UUID tbMsgId) { + this.tbMsgId = tbMsgId; + return this; + } + + public Builder tbMsgType(TbMsgType tbMsgType) { + this.tbMsgType = tbMsgType; + return this; + } + public Builder callback(FutureCallback callback) { this.callback = callback; return this; @@ -120,7 +162,10 @@ public class AttributesSaveRequest { } public AttributesSaveRequest build() { - return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, callback); + return new AttributesSaveRequest( + tenantId, entityId, scope, entries, notifyDevice, requireNonNullElse(strategy, Strategy.PROCESS_ALL), + previousCalculatedFieldIds, tbMsgId, tbMsgType, requireNonNullElse(callback, NoOpFutureCallback.instance()) + ); } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/CalculatedFieldSystemAwareRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/CalculatedFieldSystemAwareRequest.java new file mode 100644 index 0000000000..fa4c414172 --- /dev/null +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/CalculatedFieldSystemAwareRequest.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.api; + +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.msg.TbMsgType; + +import java.util.List; +import java.util.UUID; + +public interface CalculatedFieldSystemAwareRequest { + + List getPreviousCalculatedFieldIds(); + + UUID getTbMsgId(); + + TbMsgType getTbMsgType(); + +} diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceStateManager.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/DeviceStateManager.java similarity index 88% rename from rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceStateManager.java rename to rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/DeviceStateManager.java index fb3e282c9a..887f7ecaa2 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineDeviceStateManager.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/DeviceStateManager.java @@ -19,7 +19,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.TbCallback; -public interface RuleEngineDeviceStateManager { +public interface DeviceStateManager { void onDeviceConnect(TenantId tenantId, DeviceId deviceId, long connectTime, TbCallback callback); @@ -29,4 +29,6 @@ public interface RuleEngineDeviceStateManager { void onDeviceInactivity(TenantId tenantId, DeviceId deviceId, long inactivityTime, TbCallback callback); + void onDeviceInactivityTimeoutUpdate(TenantId tenantId, DeviceId deviceId, long inactivityTimeout, TbCallback callback); + } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineCalculatedFieldQueueService.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineCalculatedFieldQueueService.java new file mode 100644 index 0000000000..6ab40b79c2 --- /dev/null +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/RuleEngineCalculatedFieldQueueService.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.api; + +import com.google.common.util.concurrent.FutureCallback; + +public interface RuleEngineCalculatedFieldQueueService { + + void pushRequestToQueue(TimeseriesSaveRequest request, FutureCallback callback); + + void pushRequestToQueue(AttributesSaveRequest request, FutureCallback callback); + +} diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index d743867b84..e473df4c54 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -50,6 +50,7 @@ import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceCredentialsService; @@ -143,8 +144,7 @@ public interface TbContext { void tellFailure(TbMsg msg, Throwable th); /** - * Puts new message to queue for processing by the Root Rule Chain - * WARNING: message is put to the Main queue. To specify other queue name - use {@link #enqueue(TbMsg, String, Runnable, Consumer)} + * Puts new message to queue from TbMsg for processing by the Root Rule Chain * * @param msg - message */ @@ -280,7 +280,7 @@ public interface TbContext { DeviceCredentialsService getDeviceCredentialsService(); - RuleEngineDeviceStateManager getDeviceStateManager(); + DeviceStateManager getDeviceStateManager(); String getDeviceStateNodeRateLimitConfig(); @@ -358,6 +358,10 @@ public interface TbContext { SlackService getSlackService(); + CalculatedFieldService getCalculatedFieldService(); + + RuleEngineCalculatedFieldQueueService getCalculatedFieldQueueService(); + boolean isExternalNodeForceAck(); /** diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesDeleteRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesDeleteRequest.java index 01cad78b98..c3f6b5c74c 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesDeleteRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesDeleteRequest.java @@ -20,21 +20,27 @@ import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.ToString; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.msg.TbMsgType; import java.util.List; +import java.util.UUID; @Getter @ToString @AllArgsConstructor(access = AccessLevel.PRIVATE) -public class TimeseriesDeleteRequest { +public class TimeseriesDeleteRequest implements CalculatedFieldSystemAwareRequest { private final TenantId tenantId; private final EntityId entityId; private final List keys; private final List deleteHistoryQueries; + private final List previousCalculatedFieldIds; + private final UUID tbMsgId; + private final TbMsgType tbMsgType; private final FutureCallback> callback; public static Builder builder() { @@ -47,6 +53,9 @@ public class TimeseriesDeleteRequest { private EntityId entityId; private List keys; private List deleteHistoryQueries; + private List previousCalculatedFieldIds; + private UUID tbMsgId; + private TbMsgType tbMsgType; private FutureCallback> callback; Builder() {} @@ -71,13 +80,28 @@ public class TimeseriesDeleteRequest { return this; } + public Builder previousCalculatedFieldIds(List previousCalculatedFieldIds) { + this.previousCalculatedFieldIds = previousCalculatedFieldIds; + return this; + } + + public Builder tbMsgId(UUID tbMsgId) { + this.tbMsgId = tbMsgId; + return this; + } + + public Builder tbMsgType(TbMsgType tbMsgType) { + this.tbMsgType = tbMsgType; + return this; + } + public Builder callback(FutureCallback> callback) { this.callback = callback; return this; } public TimeseriesDeleteRequest build() { - return new TimeseriesDeleteRequest(tenantId, entityId, keys, deleteHistoryQueries, callback); + return new TimeseriesDeleteRequest(tenantId, entityId, keys, deleteHistoryQueries, previousCalculatedFieldIds, tbMsgId, tbMsgType, callback); } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java index 103b354be9..c402a0c984 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java @@ -20,18 +20,24 @@ import com.google.common.util.concurrent.SettableFuture; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; +import org.thingsboard.common.util.NoOpFutureCallback; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; import java.util.List; +import java.util.UUID; + +import static java.util.Objects.requireNonNullElse; @Getter @AllArgsConstructor(access = AccessLevel.PRIVATE) -public class TimeseriesSaveRequest { +public class TimeseriesSaveRequest implements CalculatedFieldSystemAwareRequest { private final TenantId tenantId; private final CustomerId customerId; @@ -39,14 +45,17 @@ public class TimeseriesSaveRequest { private final List entries; private final long ttl; private final Strategy strategy; + private final List previousCalculatedFieldIds; + private final UUID tbMsgId; + private final TbMsgType tbMsgType; private final FutureCallback callback; - public record Strategy(boolean saveTimeseries, boolean saveLatest, boolean sendWsUpdate) { + public record Strategy(boolean saveTimeseries, boolean saveLatest, boolean sendWsUpdate, boolean processCalculatedFields) { - public static final Strategy SAVE_ALL = new Strategy(true, true, true); - public static final Strategy WS_ONLY = new Strategy(false, false, true); - public static final Strategy LATEST_AND_WS = new Strategy(false, true, true); - public static final Strategy SKIP_ALL = new Strategy(false, false, false); + public static final Strategy PROCESS_ALL = new Strategy(true, true, true, true); + public static final Strategy WS_ONLY = new Strategy(false, false, true, false); + public static final Strategy LATEST_AND_WS = new Strategy(false, true, true, false); + public static final Strategy SKIP_ALL = new Strategy(false, false, false, false); } @@ -61,7 +70,10 @@ public class TimeseriesSaveRequest { private EntityId entityId; private List entries; private long ttl; - private Strategy strategy = Strategy.SAVE_ALL; + private Strategy strategy; + private List previousCalculatedFieldIds; + private UUID tbMsgId; + private TbMsgType tbMsgType; private FutureCallback callback; Builder() {} @@ -104,6 +116,21 @@ public class TimeseriesSaveRequest { return this; } + public Builder previousCalculatedFieldIds(List previousCalculatedFieldIds) { + this.previousCalculatedFieldIds = previousCalculatedFieldIds; + return this; + } + + public Builder tbMsgId(UUID tbMsgId) { + this.tbMsgId = tbMsgId; + return this; + } + + public Builder tbMsgType(TbMsgType tbMsgType) { + this.tbMsgType = tbMsgType; + return this; + } + public Builder callback(FutureCallback callback) { this.callback = callback; return this; @@ -124,7 +151,10 @@ public class TimeseriesSaveRequest { } public TimeseriesSaveRequest build() { - return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, strategy, callback); + return new TimeseriesSaveRequest( + tenantId, customerId, entityId, entries, ttl, requireNonNullElse(strategy, Strategy.PROCESS_ALL), + previousCalculatedFieldIds, tbMsgId, tbMsgType, requireNonNullElse(callback, NoOpFutureCallback.instance()) + ); } } diff --git a/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/AttributesDeleteRequestTest.java b/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/AttributesDeleteRequestTest.java new file mode 100644 index 0000000000..9b4a825a66 --- /dev/null +++ b/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/AttributesDeleteRequestTest.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.api; + +import org.junit.jupiter.api.Test; +import org.thingsboard.common.util.NoOpFutureCallback; + +import static org.assertj.core.api.Assertions.assertThat; + +class AttributesDeleteRequestTest { + + @Test + void testDefaultCallbackIsNoOp() { + var request = AttributesDeleteRequest.builder().build(); + + assertThat(request.getCallback()).isEqualTo(NoOpFutureCallback.instance()); + } + + @Test + void testNullCallbackIsNoOp() { + var request = AttributesDeleteRequest.builder().callback(null).build(); + + assertThat(request.getCallback()).isEqualTo(NoOpFutureCallback.instance()); + } + +} diff --git a/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/AttributesSaveRequestTest.java b/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/AttributesSaveRequestTest.java new file mode 100644 index 0000000000..d632edfbf9 --- /dev/null +++ b/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/AttributesSaveRequestTest.java @@ -0,0 +1,68 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.api; + +import org.junit.jupiter.api.Test; +import org.thingsboard.common.util.NoOpFutureCallback; + +import static org.assertj.core.api.Assertions.assertThat; + +class AttributesSaveRequestTest { + + @Test + void testDefaultProcessingStrategyIsProcessAll() { + var request = AttributesSaveRequest.builder().build(); + + assertThat(request.getStrategy()).isEqualTo(AttributesSaveRequest.Strategy.PROCESS_ALL); + } + + @Test + void testNullProcessingStrategyIsProcessAll() { + var request = AttributesSaveRequest.builder().strategy(null).build(); + + assertThat(request.getStrategy()).isEqualTo(AttributesSaveRequest.Strategy.PROCESS_ALL); + } + + @Test + void testProcessAllStrategy() { + assertThat(AttributesSaveRequest.Strategy.PROCESS_ALL).isEqualTo(new AttributesSaveRequest.Strategy(true, true, true)); + } + + @Test + void testWsOnlyStrategy() { + assertThat(AttributesSaveRequest.Strategy.WS_ONLY).isEqualTo(new AttributesSaveRequest.Strategy(false, true, false)); + } + + @Test + void testSkipAllStrategy() { + assertThat(AttributesSaveRequest.Strategy.SKIP_ALL).isEqualTo(new AttributesSaveRequest.Strategy(false, false, false)); + } + + @Test + void testDefaultCallbackIsNoOp() { + var request = AttributesSaveRequest.builder().build(); + + assertThat(request.getCallback()).isEqualTo(NoOpFutureCallback.instance()); + } + + @Test + void testNullCallbackIsNoOp() { + var request = AttributesSaveRequest.builder().callback(null).build(); + + assertThat(request.getCallback()).isEqualTo(NoOpFutureCallback.instance()); + } + +} diff --git a/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java b/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java index ecefcca1ea..c0eb04152a 100644 --- a/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java +++ b/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java @@ -16,36 +16,58 @@ package org.thingsboard.rule.engine.api; import org.junit.jupiter.api.Test; +import org.thingsboard.common.util.NoOpFutureCallback; import static org.assertj.core.api.Assertions.assertThat; class TimeseriesSaveRequestTest { @Test - void testDefaultSaveStrategyIsSaveAll() { + void testDefaultProcessingStrategyIsProcessAll() { var request = TimeseriesSaveRequest.builder().build(); - assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL); + assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.PROCESS_ALL); } @Test - void testSaveAllStrategy() { - assertThat(TimeseriesSaveRequest.Strategy.SAVE_ALL).isEqualTo(new TimeseriesSaveRequest.Strategy(true, true, true)); + void testNullProcessingStrategyIsProcessAll() { + var request = TimeseriesSaveRequest.builder().strategy(null).build(); + + assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.PROCESS_ALL); + } + + @Test + void testProcessAllStrategy() { + assertThat(TimeseriesSaveRequest.Strategy.PROCESS_ALL).isEqualTo(new TimeseriesSaveRequest.Strategy(true, true, true, true)); } @Test void testWsOnlyStrategy() { - assertThat(TimeseriesSaveRequest.Strategy.WS_ONLY).isEqualTo(new TimeseriesSaveRequest.Strategy(false, false, true)); + assertThat(TimeseriesSaveRequest.Strategy.WS_ONLY).isEqualTo(new TimeseriesSaveRequest.Strategy(false, false, true, false)); } @Test void testLatestAndWsStrategy() { - assertThat(TimeseriesSaveRequest.Strategy.LATEST_AND_WS).isEqualTo(new TimeseriesSaveRequest.Strategy(false, true, true)); + assertThat(TimeseriesSaveRequest.Strategy.LATEST_AND_WS).isEqualTo(new TimeseriesSaveRequest.Strategy(false, true, true, false)); } @Test void testSkipAllStrategy() { - assertThat(TimeseriesSaveRequest.Strategy.SKIP_ALL).isEqualTo(new TimeseriesSaveRequest.Strategy(false, false, false)); + assertThat(TimeseriesSaveRequest.Strategy.SKIP_ALL).isEqualTo(new TimeseriesSaveRequest.Strategy(false, false, false, false)); + } + + @Test + void testDefaultCallbackIsNoOp() { + var request = TimeseriesSaveRequest.builder().build(); + + assertThat(request.getCallback()).isEqualTo(NoOpFutureCallback.instance()); + } + + @Test + void testNullCallbackIsNoOp() { + var request = TimeseriesSaveRequest.builder().callback(null).build(); + + assertThat(request.getCallback()).isEqualTo(NoOpFutureCallback.instance()); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbDeviceStateNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbDeviceStateNode.java index 3aed05d372..c9a50cf88d 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbDeviceStateNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbDeviceStateNode.java @@ -17,7 +17,7 @@ package org.thingsboard.rule.engine.action; import lombok.extern.slf4j.Slf4j; import org.springframework.util.ConcurrentReferenceHashMap; -import org.thingsboard.rule.engine.api.RuleEngineDeviceStateManager; +import org.thingsboard.rule.engine.api.DeviceStateManager; import org.thingsboard.rule.engine.api.RuleNode; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNode; @@ -119,7 +119,7 @@ public class TbDeviceStateNode implements TbNode { TenantId tenantId = ctx.getTenantId(); long eventTs = msg.getMetaDataTs(); - RuleEngineDeviceStateManager deviceStateManager = ctx.getDeviceStateManager(); + DeviceStateManager deviceStateManager = ctx.getDeviceStateManager(); TbCallback callback = getMsgEnqueuedCallback(ctx, msg); switch (event) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbCalculatedFieldsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbCalculatedFieldsNode.java new file mode 100644 index 0000000000..b2cce52c87 --- /dev/null +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbCalculatedFieldsNode.java @@ -0,0 +1,125 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.telemetry; + +import com.google.gson.JsonParser; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; +import org.thingsboard.rule.engine.api.EmptyNodeConfiguration; +import org.thingsboard.rule.engine.api.RuleNode; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNode; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.server.common.adaptor.JsonConverter; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.plugin.ComponentType; +import org.thingsboard.server.common.msg.TbMsg; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.thingsboard.server.common.data.DataConstants.SCOPE; + +@Slf4j +@RuleNode( + type = ComponentType.ACTION, + name = "calculated fields", + configClazz = EmptyNodeConfiguration.class, + nodeDescription = "Pushes incoming messages to calculated fields service", + nodeDetails = "Node enables the processing of calculated fields without persisting incoming messages to the database. " + + "By default, the processing of calculated fields is triggered by the save attributes and save time series nodes. " + + "This rule node accepts the same messages as these nodes but allows you to trigger the processing of calculated " + + "fields independently, ensuring that derived data can be computed and utilized in real time without storing the original message in the database.", + configDirective = "tbNodeEmptyConfig", + icon = "published_with_changes" +) +public class TbCalculatedFieldsNode implements TbNode { + + private EmptyNodeConfiguration config; + + @Override + public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { + this.config = TbNodeUtils.convert(configuration, EmptyNodeConfiguration.class); + } + + @Override + public void onMsg(TbContext ctx, TbMsg msg) { + switch (msg.getInternalType()) { + case POST_TELEMETRY_REQUEST -> processPostTelemetryRequest(ctx, msg); + case POST_ATTRIBUTES_REQUEST -> processPostAttributesRequest(ctx, msg); + default -> ctx.tellFailure(msg, new IllegalArgumentException("Unsupported msg type: " + msg.getType())); + } + } + + private void processPostTelemetryRequest(TbContext ctx, TbMsg msg) { + Map> tsKvMap = JsonConverter.convertToTelemetry(JsonParser.parseString(msg.getData()), System.currentTimeMillis()); + + if (tsKvMap.isEmpty()) { + ctx.tellSuccess(msg); + return; + } + + List tsKvEntryList = new ArrayList<>(); + for (Map.Entry> tsKvEntry : tsKvMap.entrySet()) { + for (KvEntry kvEntry : tsKvEntry.getValue()) { + tsKvEntryList.add(new BasicTsKvEntry(tsKvEntry.getKey(), kvEntry)); + } + } + + TimeseriesSaveRequest timeseriesSaveRequest = TimeseriesSaveRequest.builder() + .tenantId(ctx.getTenantId()) + .customerId(msg.getCustomerId()) + .entityId(msg.getOriginator()) + .entries(tsKvEntryList) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) + .callback(new TelemetryNodeCallback(ctx, msg)) + .build(); + + ctx.getCalculatedFieldQueueService().pushRequestToQueue(timeseriesSaveRequest, timeseriesSaveRequest.getCallback()); + } + + private void processPostAttributesRequest(TbContext ctx, TbMsg msg) { + List newAttributes = new ArrayList<>(JsonConverter.convertToAttributes(JsonParser.parseString(msg.getData()))); + + if (newAttributes.isEmpty()) { + ctx.tellSuccess(msg); + return; + } + + AttributesSaveRequest attributesSaveRequest = AttributesSaveRequest.builder() + .tenantId(ctx.getTenantId()) + .entityId(msg.getOriginator()) + .scope(AttributeScope.valueOf(msg.getMetaData().getValue(SCOPE))) + .entries(newAttributes) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) + .callback(new TelemetryNodeCallback(ctx, msg)) + .build(); + ctx.getCalculatedFieldQueueService().pushRequestToQueue(attributesSaveRequest, attributesSaveRequest.getCallback()); + } + +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index da6d82707a..c04f5b474d 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -23,6 +23,7 @@ import com.google.common.util.concurrent.MoreExecutors; import com.google.gson.JsonParser; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.DonAsynchron; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.RuleNode; import org.thingsboard.rule.engine.api.TbContext; @@ -30,6 +31,7 @@ import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.rule.engine.telemetry.settings.AttributesProcessingSettings; import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.StringUtils; @@ -43,9 +45,14 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.UUID; import java.util.function.Function; import java.util.stream.Collectors; +import static org.thingsboard.rule.engine.telemetry.settings.AttributesProcessingSettings.Advanced; +import static org.thingsboard.rule.engine.telemetry.settings.AttributesProcessingSettings.Deduplicate; +import static org.thingsboard.rule.engine.telemetry.settings.AttributesProcessingSettings.OnEveryMessage; +import static org.thingsboard.rule.engine.telemetry.settings.AttributesProcessingSettings.WebSocketsOnly; import static org.thingsboard.server.common.data.DataConstants.NOTIFY_DEVICE_METADATA_KEY; import static org.thingsboard.server.common.data.DataConstants.SCOPE; import static org.thingsboard.server.common.data.msg.TbMsgType.POST_ATTRIBUTES_REQUEST; @@ -55,13 +62,51 @@ import static org.thingsboard.server.common.data.msg.TbMsgType.POST_ATTRIBUTES_R type = ComponentType.ACTION, name = "save attributes", configClazz = TbMsgAttributesNodeConfiguration.class, - version = 2, - nodeDescription = "Saves attributes data", - nodeDetails = "Saves entity attributes based on configurable scope parameter. Expects messages with 'POST_ATTRIBUTES_REQUEST' message type. " + - "If upsert(update/insert) operation is completed successfully rule node will send the incoming message via Success chain, otherwise, Failure chain is used. " + - "Additionally if checkbox Send attributes updated notification is set to true, rule node will put the \"Attributes Updated\" " + - "event for SHARED_SCOPE and SERVER_SCOPE attributes updates to the corresponding rule engine queue." + - "Performance checkbox 'Save attributes only if the value changes' will skip attributes overwrites for values with no changes (avoid concurrent writes because this check is not transactional; will not update 'Last updated time' for skipped attributes).", + version = 3, + nodeDescription = """ + Saves attribute data with a configurable scope and according to configured processing strategies. + """, + nodeDetails = """ + Node performs three actions: +
    +
  • Attributes: save attribute data to a database.
    • +
  • WebSockets: notify WebSockets subscriptions about attribute data updates.
    • +
  • Calculated fields: notify calculated fields about attribute data updates.
    • +
+ + For each action, three processing strategies are available: +
    +
  • On every message: perform the action for every message.
    • +
  • Deduplicate: perform the action only for the first message from a particular originator within a configurable interval.
    • +
  • Skip: never perform the action.
    • +
+ + Processing strategies are configured using processing settings, which support two modes: +
    +
  • Basic +
      +
    • On every message: applies the "On every message" strategy to all actions.
      • +
    • Deduplicate: applies the "Deduplicate" strategy (with a specified interval) to all actions.
      • +
    • WebSockets only: for all actions except WebSocket notifications, the "Skip" strategy is applied, while WebSocket notifications use the "On every message" strategy.
      • +
    +
    • +
  • Advanced: configure each action’s strategy independently.
    • +
+ + The node supports three attribute scopes: Client attributes, Shared attributes, and Server attributes. + You can set the default scope in the node configuration, or override it by specifying a valid scope property in the message metadata. +

+ Additionally: +
    +
  • If Save attributes only if the value changes is enabled, the rule node compares the received attribute value with the current stored value and skips the save operation if they match.
    • +
  • If Send attributes updated notification is enabled, the rule node will put the Attributes Updated event for SHARED_SCOPE and SERVER_SCOPE attribute updates to the queue named Main.
    • +
  • If Force notification to the device is enabled, then rule node will always notify device about SHARED_SCOPE attribute updates, regardless of the value of notifyDevice metadata property.
    • +
+ + This node expects messages of type POST_ATTRIBUTES_REQUEST. +

+ Output connections: Success, Failure. + """, configDirective = "tbActionNodeAttributesConfig", icon = "file_upload" ) @@ -73,9 +118,12 @@ public class TbMsgAttributesNode implements TbNode { private TbMsgAttributesNodeConfiguration config; + private AttributesProcessingSettings processingSettings; + @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { - this.config = TbNodeUtils.convert(configuration, TbMsgAttributesNodeConfiguration.class); + config = TbNodeUtils.convert(configuration, TbMsgAttributesNodeConfiguration.class); + processingSettings = config.getProcessingSettings(); } @Override @@ -90,11 +138,20 @@ public class TbMsgAttributesNode implements TbNode { ctx.tellSuccess(msg); return; } + + AttributesSaveRequest.Strategy strategy = determineSaveStrategy(msg.getMetaDataTs(), msg.getOriginator().getId()); + + // short-circuit + if (!strategy.saveAttributes() && !strategy.sendWsUpdate() && !strategy.processCalculatedFields()) { + ctx.tellSuccess(msg); + return; + } + AttributeScope scope = getScope(msg.getMetaData().getValue(SCOPE)); boolean sendAttributesUpdateNotification = checkSendNotification(scope); if (!config.isUpdateAttributesOnlyOnValueChange()) { - saveAttr(newAttributes, ctx, msg, scope, sendAttributesUpdateNotification); + saveAttr(newAttributes, ctx, msg, scope, sendAttributesUpdateNotification, strategy); return; } @@ -104,13 +161,42 @@ public class TbMsgAttributesNode implements TbNode { DonAsynchron.withCallback(findFuture, currentAttributes -> { List attributesChanged = filterChangedAttr(currentAttributes, newAttributes); - saveAttr(attributesChanged, ctx, msg, scope, sendAttributesUpdateNotification); + saveAttr(attributesChanged, ctx, msg, scope, sendAttributesUpdateNotification, strategy); }, throwable -> ctx.tellFailure(msg, throwable), MoreExecutors.directExecutor()); } - void saveAttr(List attributes, TbContext ctx, TbMsg msg, AttributeScope scope, boolean sendAttributesUpdateNotification) { + private AttributesSaveRequest.Strategy determineSaveStrategy(long ts, UUID originatorUuid) { + if (processingSettings instanceof OnEveryMessage) { + return AttributesSaveRequest.Strategy.PROCESS_ALL; + } + if (processingSettings instanceof WebSocketsOnly) { + return AttributesSaveRequest.Strategy.WS_ONLY; + } + if (processingSettings instanceof Deduplicate deduplicate) { + boolean isFirstMsgInInterval = deduplicate.getProcessingStrategy().shouldProcess(ts, originatorUuid); + return isFirstMsgInInterval ? AttributesSaveRequest.Strategy.PROCESS_ALL : AttributesSaveRequest.Strategy.SKIP_ALL; + } + if (processingSettings instanceof Advanced advanced) { + return new AttributesSaveRequest.Strategy( + advanced.attributes().shouldProcess(ts, originatorUuid), + advanced.webSockets().shouldProcess(ts, originatorUuid), + advanced.calculatedFields().shouldProcess(ts, originatorUuid) + ); + } + // should not happen + throw new IllegalArgumentException("Unknown processing settings type: " + processingSettings.getClass().getSimpleName()); + } + + private void saveAttr( + List attributes, + TbContext ctx, + TbMsg msg, + AttributeScope scope, + boolean sendAttributesUpdateNotification, + AttributesSaveRequest.Strategy strategy + ) { if (attributes.isEmpty()) { ctx.tellSuccess(msg); return; @@ -124,11 +210,15 @@ public class TbMsgAttributesNode implements TbNode { .scope(scope) .entries(attributes) .notifyDevice(config.isNotifyDevice() || checkNotifyDeviceMdValue(msg.getMetaData().getValue(NOTIFY_DEVICE_METADATA_KEY))) + .strategy(strategy) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) .callback(callback) .build()); } - List filterChangedAttr(List currentAttributes, List newAttributes) { + private List filterChangedAttr(List currentAttributes, List newAttributes) { if (currentAttributes == null || currentAttributes.isEmpty()) { return newAttributes; } @@ -178,6 +268,9 @@ public class TbMsgAttributesNode implements TbNode { hasChanges = fixEscapedBooleanConfigParameter(oldConfiguration, SEND_ATTRIBUTES_UPDATED_NOTIFICATION_KEY, hasChanges, false); // update updateAttributesOnlyOnValueChange. hasChanges = fixEscapedBooleanConfigParameter(oldConfiguration, UPDATE_ATTRIBUTES_ONLY_ON_VALUE_CHANGE_KEY, hasChanges, true); + case 2: + hasChanges = true; + ((ObjectNode) oldConfiguration).set("processingSettings", JacksonUtil.valueToTree(new OnEveryMessage())); break; default: break; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNodeConfiguration.java index 161aa64d5f..2687125a0c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNodeConfiguration.java @@ -15,13 +15,20 @@ */ package org.thingsboard.rule.engine.telemetry; +import jakarta.validation.constraints.NotNull; import lombok.Data; import org.thingsboard.rule.engine.api.NodeConfiguration; +import org.thingsboard.rule.engine.telemetry.settings.AttributesProcessingSettings; import org.thingsboard.server.common.data.DataConstants; +import static org.thingsboard.rule.engine.telemetry.settings.AttributesProcessingSettings.OnEveryMessage; + @Data public class TbMsgAttributesNodeConfiguration implements NodeConfiguration { + @NotNull + private AttributesProcessingSettings processingSettings; + private String scope; private boolean notifyDevice; @@ -31,6 +38,7 @@ public class TbMsgAttributesNodeConfiguration implements NodeConfigurationactions: + Node performs four actions:
  • Time series: save time series data to a ts_kv table in a DB.
  • Latest values: save time series data to a ts_kv_latest table in a DB.
  • WebSockets: notify WebSockets subscriptions about time series data updates.
    • +
  • Calculated fields: notify calculated fields about time series data updates.
For each action, three processing strategies are available: @@ -81,7 +82,7 @@ import static org.thingsboard.server.common.data.msg.TbMsgType.POST_TELEMETRY_RE
  • On every message: applies the "On every message" strategy to all actions.
  • Deduplicate: applies the "Deduplicate" strategy (with a specified interval) to all actions.
    • -
  • WebSockets only: applies the "Skip" strategy to Time series and Latest values, and the "On every message" strategy to WebSockets.
    • +
  • WebSockets only: for all actions except WebSocket notifications, the "Skip" strategy is applied, while WebSocket notifications use the "On every message" strategy.
  • Advanced: configure each action’s strategy independently.
    • @@ -90,7 +91,7 @@ import static org.thingsboard.server.common.data.msg.TbMsgType.POST_TELEMETRY_RE By default, the timestamp is taken from metadata.ts. You can enable Use server timestamp to always use the current server time instead. This is particularly useful in sequential processing scenarios where messages may arrive with out-of-order timestamps from - multiple sources. Note that the DB layer may ignore older records for attributes and latest values, + multiple sources. Note that the DB layer may ignore "outdated" records for attributes and latest values, so enabling Use server timestamp can ensure correct ordering.

    The TTL is taken first from metadata.TTL. If absent, the node configuration’s default @@ -110,7 +111,7 @@ public class TbMsgTimeseriesNode implements TbNode { private TbContext ctx; private long tenantProfileDefaultStorageTtl; - private ProcessingSettings processingSettings; + private TimeseriesProcessingSettings processingSettings; @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { @@ -137,7 +138,7 @@ public class TbMsgTimeseriesNode implements TbNode { TimeseriesSaveRequest.Strategy strategy = determineSaveStrategy(ts, msg.getOriginator().getId()); // short-circuit - if (!strategy.saveTimeseries() && !strategy.saveLatest() && !strategy.sendWsUpdate()) { + if (!strategy.saveTimeseries() && !strategy.saveLatest() && !strategy.sendWsUpdate() && !strategy.processCalculatedFields()) { ctx.tellSuccess(msg); return; } @@ -166,6 +167,9 @@ public class TbMsgTimeseriesNode implements TbNode { .entries(tsKvEntryList) .ttl(ttl) .strategy(strategy) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) .callback(new TelemetryNodeCallback(ctx, msg)) .build()); } @@ -176,20 +180,21 @@ public class TbMsgTimeseriesNode implements TbNode { private TimeseriesSaveRequest.Strategy determineSaveStrategy(long ts, UUID originatorUuid) { if (processingSettings instanceof OnEveryMessage) { - return TimeseriesSaveRequest.Strategy.SAVE_ALL; + return TimeseriesSaveRequest.Strategy.PROCESS_ALL; } if (processingSettings instanceof WebSocketsOnly) { return TimeseriesSaveRequest.Strategy.WS_ONLY; } if (processingSettings instanceof Deduplicate deduplicate) { boolean isFirstMsgInInterval = deduplicate.getProcessingStrategy().shouldProcess(ts, originatorUuid); - return isFirstMsgInInterval ? TimeseriesSaveRequest.Strategy.SAVE_ALL : TimeseriesSaveRequest.Strategy.SKIP_ALL; + return isFirstMsgInInterval ? TimeseriesSaveRequest.Strategy.PROCESS_ALL : TimeseriesSaveRequest.Strategy.SKIP_ALL; } if (processingSettings instanceof Advanced advanced) { return new TimeseriesSaveRequest.Strategy( advanced.timeseries().shouldProcess(ts, originatorUuid), advanced.latest().shouldProcess(ts, originatorUuid), - advanced.webSockets().shouldProcess(ts, originatorUuid) + advanced.webSockets().shouldProcess(ts, originatorUuid), + advanced.calculatedFields().shouldProcess(ts, originatorUuid) ); } // should not happen @@ -212,6 +217,7 @@ public class TbMsgTimeseriesNode implements TbNode { var skipLatestProcessingSettings = new Advanced( ProcessingStrategy.onEveryMessage(), ProcessingStrategy.skip(), + ProcessingStrategy.onEveryMessage(), ProcessingStrategy.onEveryMessage() ); ((ObjectNode) oldConfiguration).set("processingSettings", JacksonUtil.valueToTree(skipLatestProcessingSettings)); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java index ce41e08475..48264978cf 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java @@ -15,23 +15,12 @@ */ package org.thingsboard.rule.engine.telemetry; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; import jakarta.validation.constraints.NotNull; import lombok.Data; -import lombok.Getter; import org.thingsboard.rule.engine.api.NodeConfiguration; -import org.thingsboard.rule.engine.telemetry.strategy.ProcessingStrategy; +import org.thingsboard.rule.engine.telemetry.settings.TimeseriesProcessingSettings; -import java.util.Objects; - -import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.ProcessingSettings.Advanced; -import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.ProcessingSettings.Deduplicate; -import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.ProcessingSettings.OnEveryMessage; -import static org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNodeConfiguration.ProcessingSettings.WebSocketsOnly; +import static org.thingsboard.rule.engine.telemetry.settings.TimeseriesProcessingSettings.OnEveryMessage; @Data public class TbMsgTimeseriesNodeConfiguration implements NodeConfiguration { @@ -39,7 +28,7 @@ public class TbMsgTimeseriesNodeConfiguration implements NodeConfiguration callbackCaptor; private TbDeviceStateNode node; @@ -263,7 +263,7 @@ public class TbDeviceStateNodeTest { @ParameterizedTest @MethodSource - public void givenSupportedEventAndDeviceOriginator_whenOnMsg_thenCorrectEventIsSentWithCorrectCallback(TbMsgType supportedEventType, BiConsumer> actionVerification) { + public void givenSupportedEventAndDeviceOriginator_whenOnMsg_thenCorrectEventIsSentWithCorrectCallback(TbMsgType supportedEventType, BiConsumer> actionVerification) { // GIVEN given(ctxMock.getTenantId()).willReturn(TENANT_ID); given(ctxMock.getDeviceStateNodeRateLimitConfig()).willReturn("1:1"); @@ -297,10 +297,10 @@ public class TbDeviceStateNodeTest { private static Stream givenSupportedEventAndDeviceOriginator_whenOnMsg_thenCorrectEventIsSentWithCorrectCallback() { return Stream.of( - Arguments.of(TbMsgType.CONNECT_EVENT, (BiConsumer>) (deviceStateManagerMock, callbackCaptor) -> then(deviceStateManagerMock).should().onDeviceConnect(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())), - Arguments.of(TbMsgType.ACTIVITY_EVENT, (BiConsumer>) (deviceStateManagerMock, callbackCaptor) -> then(deviceStateManagerMock).should().onDeviceActivity(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())), - Arguments.of(TbMsgType.DISCONNECT_EVENT, (BiConsumer>) (deviceStateManagerMock, callbackCaptor) -> then(deviceStateManagerMock).should().onDeviceDisconnect(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())), - Arguments.of(TbMsgType.INACTIVITY_EVENT, (BiConsumer>) (deviceStateManagerMock, callbackCaptor) -> then(deviceStateManagerMock).should().onDeviceInactivity(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())) + Arguments.of(TbMsgType.CONNECT_EVENT, (BiConsumer>) (deviceStateManagerMock, callbackCaptor) -> then(deviceStateManagerMock).should().onDeviceConnect(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())), + Arguments.of(TbMsgType.ACTIVITY_EVENT, (BiConsumer>) (deviceStateManagerMock, callbackCaptor) -> then(deviceStateManagerMock).should().onDeviceActivity(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())), + Arguments.of(TbMsgType.DISCONNECT_EVENT, (BiConsumer>) (deviceStateManagerMock, callbackCaptor) -> then(deviceStateManagerMock).should().onDeviceDisconnect(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())), + Arguments.of(TbMsgType.INACTIVITY_EVENT, (BiConsumer>) (deviceStateManagerMock, callbackCaptor) -> then(deviceStateManagerMock).should().onDeviceInactivity(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())) ); } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java index d094035d3b..71e9ed8350 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java @@ -533,7 +533,7 @@ public class TbMathNodeTest { verify(ctx, timeout(TIMEOUT)).tellSuccess(msgCaptor.capture()); verify(telemetryService, times(1)).saveTimeseries(assertArg(request -> { assertThat(request.getEntries()).size().isOne(); - assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL); + assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.PROCESS_ALL); })); TbMsg resultMsg = msgCaptor.getValue(); @@ -569,7 +569,7 @@ public class TbMathNodeTest { verify(ctx, timeout(TIMEOUT)).tellSuccess(msgCaptor.capture()); verify(telemetryService, times(1)).saveTimeseries(assertArg(request -> { assertThat(request.getEntries()).size().isOne(); - assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL); + assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.PROCESS_ALL); })); TbMsg resultMsg = msgCaptor.getValue(); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNodeTest.java index a684d23b73..941dfb92d7 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNodeTest.java @@ -15,13 +15,15 @@ */ package org.thingsboard.rule.engine.telemetry; -import com.fasterxml.jackson.databind.node.ObjectNode; -import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.AbstractRuleNodeUpgradeTest; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; @@ -29,6 +31,7 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.telemetry.strategy.ProcessingStrategy; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; @@ -42,185 +45,660 @@ import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.service.ConstraintValidator; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.stream.Stream; +import static com.google.common.util.concurrent.Futures.immediateFuture; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.assertArg; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.BDDMockito.willCallRealMethod; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; +import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.then; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.thingsboard.rule.engine.api.AttributesSaveRequest.Strategy; +import static org.thingsboard.rule.engine.api.AttributesSaveRequest.builder; +import static org.thingsboard.rule.engine.telemetry.settings.AttributesProcessingSettings.Advanced; +import static org.thingsboard.rule.engine.telemetry.settings.AttributesProcessingSettings.Deduplicate; +import static org.thingsboard.rule.engine.telemetry.settings.AttributesProcessingSettings.OnEveryMessage; +import static org.thingsboard.rule.engine.telemetry.settings.AttributesProcessingSettings.WebSocketsOnly; import static org.thingsboard.server.common.data.DataConstants.NOTIFY_DEVICE_METADATA_KEY; -@Slf4j +@ExtendWith(MockitoExtension.class) class TbMsgAttributesNodeTest extends AbstractRuleNodeUpgradeTest { - private TenantId tenantId; - private DeviceId deviceId; - private TbMsgAttributesNode node; + final TenantId tenantId = TenantId.fromUUID(UUID.fromString("6c18691e-4470-4766-9739-aface71d761f")); + final DeviceId deviceId = new DeviceId(UUID.fromString("b66159d7-c77e-45e8-bb41-a8f557f434c1")); + + @Spy + TbMsgAttributesNode node; + TbMsgAttributesNodeConfiguration config; + + @Mock + TbContext ctxMock; + @Mock + AttributesService attributesServiceMock; + @Mock + RuleEngineTelemetryService telemetryServiceMock; @BeforeEach void setUp() { - tenantId = new TenantId(UUID.fromString("6c18691e-4470-4766-9739-aface71d761f")); - deviceId = new DeviceId(UUID.fromString("b66159d7-c77e-45e8-bb41-a8f557f434c1")); - node = spy(TbMsgAttributesNode.class); + lenient().when(ctxMock.getTenantId()).thenReturn(tenantId); + lenient().when(ctxMock.getAttributesService()).thenReturn(attributesServiceMock); + lenient().when(ctxMock.getTelemetryService()).thenReturn(telemetryServiceMock); + + config = new TbMsgAttributesNodeConfiguration().defaultConfiguration(); + } + + @Test + void verifyDefaultConfig() { + assertThat(config.getProcessingSettings()).isInstanceOf(OnEveryMessage.class); + assertThat(config.getScope()).isEqualTo("SERVER_SCOPE"); + assertThat(config.isNotifyDevice()).isFalse(); + assertThat(config.isSendAttributesUpdatedNotification()).isFalse(); + assertThat(config.isUpdateAttributesOnlyOnValueChange()).isTrue(); } @Test - void testFilterChangedAttr_whenCurrentAttributesEmpty_thenReturnNewAttributes() { - List newAttributes = new ArrayList<>(); + void givenProcessingSettingsAreNull_whenValidatingConstraints_thenThrowsException() { + // GIVEN + config.setProcessingSettings(null); - List filtered = node.filterChangedAttr(Collections.emptyList(), newAttributes); - assertThat(filtered).isSameAs(newAttributes); + // WHEN-THEN + assertThatThrownBy(() -> ConstraintValidator.validateFields(config)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Validation error: processingSettings must not be null"); } @Test - void testFilterChangedAttr_whenCurrentAttributesContainsInAnyOrderNewAttributes_thenReturnEmptyList() { - List currentAttributes = List.of( - new BaseAttributeKvEntry(1694000000L, new StringDataEntry("address", "Peremohy ave 1")), - new BaseAttributeKvEntry(1694000000L, new BooleanDataEntry("valid", true)), - new BaseAttributeKvEntry(1694000000L, new LongDataEntry("counter", 100L)), - new BaseAttributeKvEntry(1694000000L, new DoubleDataEntry("temp", -18.35)), - new BaseAttributeKvEntry(1694000000L, new JsonDataEntry("json", "{\"warning\":\"out of paper\"}")) - ); - List newAttributes = new ArrayList<>(currentAttributes); - newAttributes.add(newAttributes.get(0)); - newAttributes.remove(0); - assertThat(newAttributes).hasSize(currentAttributes.size()); - assertThat(currentAttributes).isNotEmpty(); - assertThat(newAttributes).containsExactlyInAnyOrderElementsOf(currentAttributes); - - List filtered = node.filterChangedAttr(currentAttributes, newAttributes); - assertThat(filtered).isEmpty(); //no changes + void givenOnEveryMessageProcessingSettingsAndSameMessageTwoTimes_whenOnMsg_thenPersistSameMessageTwoTimes() throws TbNodeException { + // GIVEN + config.setUpdateAttributesOnlyOnValueChange(false); + config.setProcessingSettings(new OnEveryMessage()); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_ATTRIBUTES_REQUEST) + .originator(deviceId) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of(NOTIFY_DEVICE_METADATA_KEY, "false"))) + .build(); + + // WHEN-THEN + var expectedSaveRequest = builder() + .tenantId(tenantId) + .entityId(msg.getOriginator()) + .scope(AttributeScope.valueOf(config.getScope())) + .entry(new DoubleDataEntry("temperature", 22.3)) + .notifyDevice(false) + .strategy(Strategy.PROCESS_ALL) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) + .build(); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(times(1)).saveAttributes(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest) + .usingRecursiveComparison() + .ignoringFields("callback", "entries.lastUpdateTs") + .isEqualTo(expectedSaveRequest) + )); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(times(2)).saveAttributes(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest) + .usingRecursiveComparison() + .ignoringFields("callback", "entries.lastUpdateTs") + .isEqualTo(expectedSaveRequest) + )); + } + + @Test + void givenDeduplicateProcessingSettingsAndSameMessageTwoTimes_whenOnMsg_thenPersistThisMessageOnlyFirstTime() throws TbNodeException { + // GIVEN + config.setUpdateAttributesOnlyOnValueChange(false); + config.setProcessingSettings(new Deduplicate(10)); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_ATTRIBUTES_REQUEST) + .originator(deviceId) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of(NOTIFY_DEVICE_METADATA_KEY, "false"))) + .build(); + + // WHEN-THEN + var expectedSaveRequest = builder() + .tenantId(tenantId) + .entityId(msg.getOriginator()) + .scope(AttributeScope.valueOf(config.getScope())) + .entry(new DoubleDataEntry("temperature", 22.3)) + .notifyDevice(false) + .strategy(Strategy.PROCESS_ALL) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) + .build(); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should().saveAttributes(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest) + .usingRecursiveComparison() + .ignoringFields("callback", "entries.lastUpdateTs") + .isEqualTo(expectedSaveRequest) + )); + + clearInvocations(telemetryServiceMock, ctxMock); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(never()).saveAttributes(any()); + } + + @Test + void givenWebSocketsOnlyProcessingSettingsAndSameMessageTwoTimes_whenOnMsg_thenSendsOnlyWsUpdateTwoTimes() throws TbNodeException { + // GIVEN + config.setUpdateAttributesOnlyOnValueChange(false); + config.setProcessingSettings(new WebSocketsOnly()); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_ATTRIBUTES_REQUEST) + .originator(deviceId) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of(NOTIFY_DEVICE_METADATA_KEY, "false"))) + .build(); + + // WHEN-THEN + var expectedSaveRequest = builder() + .tenantId(tenantId) + .entityId(msg.getOriginator()) + .scope(AttributeScope.valueOf(config.getScope())) + .entry(new DoubleDataEntry("temperature", 22.3)) + .notifyDevice(false) + .strategy(Strategy.WS_ONLY) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) + .build(); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(times(1)).saveAttributes(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest) + .usingRecursiveComparison() + .ignoringFields("callback", "entries.lastUpdateTs") + .isEqualTo(expectedSaveRequest) + )); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(times(2)).saveAttributes(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest) + .usingRecursiveComparison() + .ignoringFields("callback", "entries.lastUpdateTs") + .isEqualTo(expectedSaveRequest) + )); + } + + @Test + void givenAdvancedProcessingSettingsWithOnEveryMessageStrategiesForAllActionsAndSameMessageTwoTimes_whenOnMsg_thenPersistSameMessageTwoTimes() throws TbNodeException { + // GIVEN + config.setUpdateAttributesOnlyOnValueChange(false); + config.setProcessingSettings(new Advanced( + ProcessingStrategy.onEveryMessage(), + ProcessingStrategy.onEveryMessage(), + ProcessingStrategy.onEveryMessage() + )); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_ATTRIBUTES_REQUEST) + .originator(deviceId) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of(NOTIFY_DEVICE_METADATA_KEY, "false"))) + .build(); + + // WHEN-THEN + var expectedSaveRequest = builder() + .tenantId(tenantId) + .entityId(msg.getOriginator()) + .scope(AttributeScope.valueOf(config.getScope())) + .entry(new DoubleDataEntry("temperature", 22.3)) + .notifyDevice(false) + .strategy(Strategy.PROCESS_ALL) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) + .build(); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(times(1)).saveAttributes(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest) + .usingRecursiveComparison() + .ignoringFields("callback", "entries.lastUpdateTs") + .isEqualTo(expectedSaveRequest) + )); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(times(2)).saveAttributes(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest) + .usingRecursiveComparison() + .ignoringFields("callback", "entries.lastUpdateTs") + .isEqualTo(expectedSaveRequest) + )); + } + + @Test + void givenAdvancedProcessingSettingsWithDifferentDeduplicateStrategyForEachAction_whenOnMsg_thenEvaluatesStrategiesForEachActionsIndependently() throws TbNodeException { + // GIVEN + config.setUpdateAttributesOnlyOnValueChange(false); + config.setProcessingSettings(new Advanced( + ProcessingStrategy.deduplicate(1), + ProcessingStrategy.deduplicate(2), + ProcessingStrategy.deduplicate(3) + )); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + long ts1 = 500L; + long ts2 = 1500L; + long ts3 = 2500L; + long ts4 = 3500L; + + // WHEN-THEN + node.onMsg(ctxMock, TbMsg.newMsg() + .type(TbMsgType.POST_ATTRIBUTES_REQUEST) + .originator(deviceId) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts1)))) + .build()); + then(telemetryServiceMock).should().saveAttributes(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo(Strategy.PROCESS_ALL) + )); + + clearInvocations(telemetryServiceMock); + + node.onMsg(ctxMock, TbMsg.newMsg() + .type(TbMsgType.POST_ATTRIBUTES_REQUEST) + .originator(deviceId) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts2)))) + .build()); + then(telemetryServiceMock).should().saveAttributes(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo(new Strategy(true, false, false)) + )); + + clearInvocations(telemetryServiceMock); + + node.onMsg(ctxMock, TbMsg.newMsg() + .type(TbMsgType.POST_ATTRIBUTES_REQUEST) + .originator(deviceId) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts3)))) + .build()); + then(telemetryServiceMock).should().saveAttributes(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo(new Strategy(true, true, false)) + )); + + clearInvocations(telemetryServiceMock); + + node.onMsg(ctxMock, TbMsg.newMsg() + .type(TbMsgType.POST_ATTRIBUTES_REQUEST) + .originator(deviceId) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts4)))) + .build()); + then(telemetryServiceMock).should().saveAttributes(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo(new Strategy(true, false, true)) + )); } @Test - void testFilterChangedAttr_whenCurrentAttributesContainsInAnyOrderNewAttributes_thenReturnExpectedList() { + public void givenAdvancedProcessingSettingsWithSkipStrategiesForAllActionsAndSameMessageTwoTimes_whenOnMsg_thenSkipsSameMessageTwoTimes() throws TbNodeException { + // GIVEN + config.setProcessingSettings(new Advanced( + ProcessingStrategy.skip(), + ProcessingStrategy.skip(), + ProcessingStrategy.skip() + )); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_ATTRIBUTES_REQUEST) + .originator(deviceId) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of(NOTIFY_DEVICE_METADATA_KEY, "false"))) + .build(); + + // WHEN-THEN + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(never()).saveAttributes(any()); + then(ctxMock).should(times(1)).tellSuccess(msg); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(never()).saveAttributes(any()); + then(ctxMock).should(times(2)).tellSuccess(msg); + } + + @Test + void givenVariousChangesToAttributes_whenUpdateOnlyOnValueChangeEnabled_thenShouldCorrectlyFilterChangedAttributes() throws TbNodeException { + // GIVEN + config.setUpdateAttributesOnlyOnValueChange(true); + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + List currentAttributes = List.of( - new BaseAttributeKvEntry(1694000000L, new StringDataEntry("address", "Peremohy ave 1")), - new BaseAttributeKvEntry(1694000000L, new BooleanDataEntry("valid", true)), - new BaseAttributeKvEntry(1694000000L, new LongDataEntry("counter", 100L)), - new BaseAttributeKvEntry(1694000000L, new DoubleDataEntry("temp", -18.35)), - new BaseAttributeKvEntry(1694000000L, new JsonDataEntry("json", "{\"warning\":\"out of paper\"}")) + new BaseAttributeKvEntry(123L, new StringDataEntry("address", "Prospect Beresteiskyi 1")), + new BaseAttributeKvEntry(123L, new BooleanDataEntry("valid", true)), + new BaseAttributeKvEntry(123L, new LongDataEntry("counter", 100L)), + new BaseAttributeKvEntry(123L, new DoubleDataEntry("temp", -18.35)), + new BaseAttributeKvEntry(123L, new JsonDataEntry("json", "{\"warning\":\"out of paper\"}")) ); - List newAttributes = List.of( - new BaseAttributeKvEntry(1694000999L, new JsonDataEntry("json", "{\"status\":\"OK\"}")), // value changed, reordered - new BaseAttributeKvEntry(1694000999L, new StringDataEntry("valid", "true")), //type changed - new BaseAttributeKvEntry(1694000999L, new LongDataEntry("counter", 101L)), //value changed - new BaseAttributeKvEntry(1694000999L, new DoubleDataEntry("temp", -18.35)), - new BaseAttributeKvEntry(1694000999L, new StringDataEntry("address", "Peremohy ave 1")) // reordered - ); - List expected = List.of( - new BaseAttributeKvEntry(1694000999L, new StringDataEntry("valid", "true")), - new BaseAttributeKvEntry(1694000999L, new LongDataEntry("counter", 101L)), - new BaseAttributeKvEntry(1694000999L, new JsonDataEntry("json", "{\"status\":\"OK\"}")) + given(attributesServiceMock.find(eq(tenantId), eq(deviceId), eq(AttributeScope.valueOf(config.getScope())), anyList())).willReturn(immediateFuture(currentAttributes)); + + var data = JacksonUtil.newObjectNode() + .put("address", "Prospect Beresteiskyi 1") // no changes + .put("valid", "false") // type and value changed + .put("counter", 101L) // value changed + .put("temp", -18.35) // no changes + .put("json", "{\"warning\":\"out of paper\"}") // only type changed + .put("newKey", "newValue"); // new attribute + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_ATTRIBUTES_REQUEST) + .originator(deviceId) + .data(data.toString()) + .metaData(TbMsgMetaData.EMPTY) + .build(); + + // WHEN + node.onMsg(ctxMock, msg); + + // THEN + List expectedChangedAttributes = List.of( + new BaseAttributeKvEntry(456L, new StringDataEntry("valid", "false")), + new BaseAttributeKvEntry(456L, new LongDataEntry("counter", 101L)), + new BaseAttributeKvEntry(456L, new StringDataEntry("json", "{\"warning\":\"out of paper\"}")), + new BaseAttributeKvEntry(456L, new StringDataEntry("newKey", "newValue")) ); - List filtered = node.filterChangedAttr(currentAttributes, newAttributes); - assertThat(filtered).containsExactlyInAnyOrderElementsOf(expected); + then(telemetryServiceMock).should().saveAttributes(assertArg(request -> + assertThat(request.getEntries()) + .usingRecursiveComparison() + .ignoringCollectionOrder() + .ignoringFields("lastUpdateTs") + .isEqualTo(expectedChangedAttributes) + )); } - // Notify device backward-compatibility test arguments - private static Stream givenNotifyDeviceMdValue_whenSaveAndNotify_thenVerifyExpectedArgumentForNotifyDeviceInSaveAndNotifyMethod() { - return Stream.of( - Arguments.of(null, true), - Arguments.of("null", false), - Arguments.of("true", true), - Arguments.of("false", false) + @Test + void givenNoChangesToAttributes_whenUpdateOnlyOnValueChangeEnabled_thenShouldNotCallSaveAndJustTellSuccess() throws TbNodeException { + // GIVEN + config.setUpdateAttributesOnlyOnValueChange(true); + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + List currentAttributes = List.of( + new BaseAttributeKvEntry(123L, new StringDataEntry("address", "Prospect Beresteiskyi 1")), + new BaseAttributeKvEntry(123L, new BooleanDataEntry("valid", true)), + new BaseAttributeKvEntry(123L, new LongDataEntry("counter", 100L)) ); + given(attributesServiceMock.find(eq(tenantId), eq(deviceId), eq(AttributeScope.valueOf(config.getScope())), anyList())).willReturn(immediateFuture(currentAttributes)); + + var data = JacksonUtil.newObjectNode() + .put("address", "Prospect Beresteiskyi 1") + .put("valid", true) + .put("counter", 100L); + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_ATTRIBUTES_REQUEST) + .originator(deviceId) + .data(data.toString()) + .metaData(TbMsgMetaData.EMPTY) + .build(); + + // WHEN + node.onMsg(ctxMock, msg); + + // THEN + then(telemetryServiceMock).shouldHaveNoInteractions(); + then(ctxMock).should().tellSuccess(msg); } // Notify device backward-compatibility test @ParameterizedTest @MethodSource - void givenNotifyDeviceMdValue_whenSaveAndNotify_thenVerifyExpectedArgumentForNotifyDeviceInSaveAndNotifyMethod(String mdValue, boolean expectedArgumentValue) throws TbNodeException { - var ctxMock = mock(TbContext.class); - var telemetryServiceMock = mock(RuleEngineTelemetryService.class); - ObjectNode defaultConfig = (ObjectNode) JacksonUtil.valueToTree(new TbMsgAttributesNodeConfiguration().defaultConfiguration()); - defaultConfig.put("notifyDevice", false); - var tbNodeConfiguration = new TbNodeConfiguration(defaultConfig); - - assertThat(defaultConfig.has("notifyDevice")).as("pre condition has notifyDevice").isTrue(); - - when(ctxMock.getTenantId()).thenReturn(tenantId); - when(ctxMock.getTelemetryService()).thenReturn(telemetryServiceMock); - willCallRealMethod().given(node).init(any(TbContext.class), any(TbNodeConfiguration.class)); - willCallRealMethod().given(node).saveAttr(any(), eq(ctxMock), any(TbMsg.class), any(AttributeScope.class), anyBoolean()); - - node.init(ctxMock, tbNodeConfiguration); - - TbMsgMetaData md = new TbMsgMetaData(); - if (mdValue != null) { - md.putValue(NOTIFY_DEVICE_METADATA_KEY, mdValue); - } - // dummy list with one ts kv to pass the empty list check. - var testTbMsg = TbMsg.newMsg() - .type(TbMsgType.POST_TELEMETRY_REQUEST) + void givenVariousValuesForNotifyDeviceInMetadata_thenShouldCorrectlyParseValueFromMetadata(String mdValue, boolean expectedArgumentValue) throws TbNodeException { + // GIVEN + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + given(attributesServiceMock.find(tenantId, deviceId, AttributeScope.valueOf(config.getScope()), List.of("mode"))).willReturn( + immediateFuture(List.of(new BaseAttributeKvEntry(123L, new StringDataEntry("mode", "tilt")))) + ); + + var metadata = new TbMsgMetaData(); + metadata.putValue(NOTIFY_DEVICE_METADATA_KEY, mdValue); + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_ATTRIBUTES_REQUEST) .originator(deviceId) - .copyMetaData(md) - .data(TbMsg.EMPTY_STRING) + .data(JacksonUtil.newObjectNode().put("mode", "vibration").toString()) + .metaData(metadata) .build(); - List testAttrList = List.of(new BaseAttributeKvEntry(0L, new StringDataEntry("testKey", "testValue"))); - node.saveAttr(testAttrList, ctxMock, testTbMsg, AttributeScope.SHARED_SCOPE, false); + // WHEN + node.onMsg(ctxMock, msg); + + // THEN + then(telemetryServiceMock).should().saveAttributes(assertArg(request -> assertThat(request.isNotifyDevice()).isEqualTo(expectedArgumentValue))); + } - verify(telemetryServiceMock, times(1)).saveAttributes(assertArg(request -> { - assertThat(request.getTenantId()).isEqualTo(tenantId); - assertThat(request.getEntityId()).isEqualTo(deviceId); - assertThat(request.getScope()).isEqualTo(AttributeScope.SHARED_SCOPE); - assertThat(request.getEntries()).isEqualTo(testAttrList); - assertThat(request.isNotifyDevice()).isEqualTo(expectedArgumentValue); - })); + // Notify device backward-compatibility test arguments + static Stream givenVariousValuesForNotifyDeviceInMetadata_thenShouldCorrectlyParseValueFromMetadata() { + return Stream.of( + Arguments.of(null, true), + Arguments.of("null", false), + Arguments.of("true", true), + Arguments.of("false", false) + ); } // Rule nodes upgrade - private static Stream givenFromVersionAndConfig_whenUpgrade_thenVerifyHasChangesAndConfig() { + static Stream givenFromVersionAndConfig_whenUpgrade_thenVerifyHasChangesAndConfig() { return Stream.of( // default config for version 0 Arguments.of(0, - "{\"scope\":\"CLIENT_SCOPE\",\"notifyDevice\":\"false\",\"sendAttributesUpdatedNotification\":\"false\"}", + """ + { + "scope": "CLIENT_SCOPE", + "notifyDevice": "false", + "sendAttributesUpdatedNotification": "false" + } + """, true, - "{\"scope\":\"CLIENT_SCOPE\",\"notifyDevice\":false,\"sendAttributesUpdatedNotification\":false,\"updateAttributesOnlyOnValueChange\":false}"), + """ + { + "processingSettings": { + "type": "ON_EVERY_MESSAGE" + }, + "scope": "CLIENT_SCOPE", + "notifyDevice": false, + "sendAttributesUpdatedNotification": false, + "updateAttributesOnlyOnValueChange": false + } + """ + ), // default config for version 1 with upgrade from version 0 Arguments.of(0, - "{\"scope\":\"CLIENT_SCOPE\",\"notifyDevice\":false,\"sendAttributesUpdatedNotification\":false,\"updateAttributesOnlyOnValueChange\":true}", - false, - "{\"scope\":\"CLIENT_SCOPE\",\"notifyDevice\":false,\"sendAttributesUpdatedNotification\":false,\"updateAttributesOnlyOnValueChange\":true}"), + """ + { + "scope": "CLIENT_SCOPE", + "notifyDevice": false, + "sendAttributesUpdatedNotification": false, + "updateAttributesOnlyOnValueChange": true + } + """, + true, + """ + { + "processingSettings": { + "type": "ON_EVERY_MESSAGE" + }, + "scope": "CLIENT_SCOPE", + "notifyDevice": false, + "sendAttributesUpdatedNotification": false, + "updateAttributesOnlyOnValueChange": true + } + """ + ), // all flags are booleans Arguments.of(1, - "{\"scope\":\"SHARED_SCOPE\",\"notifyDevice\":true,\"sendAttributesUpdatedNotification\":false,\"updateAttributesOnlyOnValueChange\":true}", - false, - "{\"scope\":\"SHARED_SCOPE\",\"notifyDevice\":true,\"sendAttributesUpdatedNotification\":false,\"updateAttributesOnlyOnValueChange\":true}"), + """ + { + "scope": "SHARED_SCOPE", + "notifyDevice": true, + "sendAttributesUpdatedNotification": false, + "updateAttributesOnlyOnValueChange": true + } + """, + true, + """ + { + "processingSettings": { + "type": "ON_EVERY_MESSAGE" + }, + "scope": "SHARED_SCOPE", + "notifyDevice": true, + "sendAttributesUpdatedNotification": false, + "updateAttributesOnlyOnValueChange": true + } + """ + ), // no boolean flags set Arguments.of(1, - "{\"scope\":\"CLIENT_SCOPE\"}", + """ + { + "scope": "CLIENT_SCOPE" + } + """, true, - "{\"scope\":\"CLIENT_SCOPE\",\"notifyDevice\":true,\"sendAttributesUpdatedNotification\":false,\"updateAttributesOnlyOnValueChange\":true}"), + """ + { + "processingSettings": { + "type": "ON_EVERY_MESSAGE" + }, + "scope": "CLIENT_SCOPE", + "notifyDevice": true, + "sendAttributesUpdatedNotification": false, + "updateAttributesOnlyOnValueChange": true + } + """ + ), // all flags are boolean strings Arguments.of(1, - "{\"scope\":\"CLIENT_SCOPE\",\"notifyDevice\":\"false\",\"sendAttributesUpdatedNotification\":\"false\",\"updateAttributesOnlyOnValueChange\":\"true\"}", + """ + { + "scope": "CLIENT_SCOPE", + "notifyDevice": "false", + "sendAttributesUpdatedNotification": "false", + "updateAttributesOnlyOnValueChange": "true" + } + """, true, - "{\"scope\":\"CLIENT_SCOPE\",\"notifyDevice\":false,\"sendAttributesUpdatedNotification\":false,\"updateAttributesOnlyOnValueChange\":true}"), + """ + { + "processingSettings": { + "type": "ON_EVERY_MESSAGE" + }, + "scope": "CLIENT_SCOPE", + "notifyDevice": false, + "sendAttributesUpdatedNotification": false, + "updateAttributesOnlyOnValueChange": true + } + """ + ), // at least one flag is boolean string Arguments.of(1, - "{\"scope\":\"CLIENT_SCOPE\",\"notifyDevice\":\"false\",\"sendAttributesUpdatedNotification\":false,\"updateAttributesOnlyOnValueChange\":true}", + """ + { + "scope": "CLIENT_SCOPE", + "notifyDevice": "false", + "sendAttributesUpdatedNotification": false, + "updateAttributesOnlyOnValueChange": true + } + """, true, - "{\"scope\":\"CLIENT_SCOPE\",\"notifyDevice\":false,\"sendAttributesUpdatedNotification\":false,\"updateAttributesOnlyOnValueChange\":true}"), + """ + { + "processingSettings": { + "type": "ON_EVERY_MESSAGE" + }, + "scope": "CLIENT_SCOPE", + "notifyDevice": false, + "sendAttributesUpdatedNotification": false, + "updateAttributesOnlyOnValueChange": true + } + """ + ), // notify device flag is null Arguments.of(1, - "{\"scope\":\"CLIENT_SCOPE\",\"notifyDevice\":\"null\",\"sendAttributesUpdatedNotification\":false,\"updateAttributesOnlyOnValueChange\":true}", + """ + { + "scope": "CLIENT_SCOPE", + "notifyDevice": "null", + "sendAttributesUpdatedNotification": false, + "updateAttributesOnlyOnValueChange": true + } + """, + true, + """ + { + "processingSettings": { + "type": "ON_EVERY_MESSAGE" + }, + "scope": "CLIENT_SCOPE", + "notifyDevice": true, + "sendAttributesUpdatedNotification": false, + "updateAttributesOnlyOnValueChange": true + } + """ + ), + // default config for version 2 + Arguments.of(2, + """ + { + "scope": "SERVER_SCOPE", + "notifyDevice": false, + "sendAttributesUpdatedNotification": false, + "updateAttributesOnlyOnValueChange": true + } + """, true, - "{\"scope\":\"CLIENT_SCOPE\",\"notifyDevice\":true,\"sendAttributesUpdatedNotification\":false,\"updateAttributesOnlyOnValueChange\":true}") + """ + { + "processingSettings": { + "type": "ON_EVERY_MESSAGE" + }, + "scope": "SERVER_SCOPE", + "notifyDevice": false, + "sendAttributesUpdatedNotification": false, + "updateAttributesOnlyOnValueChange": true + } + """ + ) ); } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java index de651e9841..1141ff09d1 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java @@ -73,6 +73,10 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.thingsboard.rule.engine.telemetry.settings.TimeseriesProcessingSettings.Advanced; +import static org.thingsboard.rule.engine.telemetry.settings.TimeseriesProcessingSettings.Deduplicate; +import static org.thingsboard.rule.engine.telemetry.settings.TimeseriesProcessingSettings.OnEveryMessage; +import static org.thingsboard.rule.engine.telemetry.settings.TimeseriesProcessingSettings.WebSocketsOnly; @ExtendWith(MockitoExtension.class) public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { @@ -110,7 +114,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { @Test public void verifyDefaultConfig() { assertThat(config.getDefaultTTL()).isEqualTo(0L); - assertThat(config.getProcessingSettings()).isInstanceOf(TbMsgTimeseriesNodeConfiguration.ProcessingSettings.OnEveryMessage.class); + assertThat(config.getProcessingSettings()).isInstanceOf(OnEveryMessage.class); assertThat(config.isUseServerTs()).isFalse(); } @@ -208,7 +212,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { assertThat(request.getEntityId()).isEqualTo(DEVICE_ID); assertThat(request.getEntries()).usingRecursiveFieldByFieldElementComparatorIgnoringFields("ts").containsExactlyElementsOf(expectedList); assertThat(request.getTtl()).isEqualTo(extractTtlAsSeconds(tenantProfile)); - assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL); + assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.PROCESS_ALL); assertThat(request.getCallback()).isInstanceOf(TelemetryNodeCallback.class); })); verify(ctxMock).tellSuccess(msg); @@ -220,10 +224,11 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { // GIVEN config.setDefaultTTL(10L); - var timeseriesStrategy = ProcessingStrategy.onEveryMessage(); - var latestStrategy = ProcessingStrategy.skip(); + var timeseries = ProcessingStrategy.onEveryMessage(); + var latest = ProcessingStrategy.skip(); var webSockets = ProcessingStrategy.onEveryMessage(); - var processingSettings = new TbMsgTimeseriesNodeConfiguration.ProcessingSettings.Advanced(timeseriesStrategy, latestStrategy, webSockets); + var calculatedFields = ProcessingStrategy.onEveryMessage(); + var processingSettings = new Advanced(timeseries, latest, webSockets, calculatedFields); config.setProcessingSettings(processingSettings); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); @@ -265,7 +270,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { assertThat(request.getEntityId()).isEqualTo(DEVICE_ID); assertThat(request.getEntries()).containsExactlyElementsOf(expectedList); assertThat(request.getTtl()).isEqualTo(config.getDefaultTTL()); - assertThat(request.getStrategy()).isEqualTo(new TimeseriesSaveRequest.Strategy(true, false, true)); + assertThat(request.getStrategy()).isEqualTo(new TimeseriesSaveRequest.Strategy(true, false, true, true)); assertThat(request.getCallback()).isInstanceOf(TelemetryNodeCallback.class); })); verify(ctxMock).tellSuccess(msg); @@ -304,7 +309,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { assertThat(request.getCustomerId()).isNull(); assertThat(request.getEntityId()).isEqualTo(DEVICE_ID); assertThat(request.getTtl()).isEqualTo(expectedTtl); - assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL); + assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.PROCESS_ALL); assertThat(request.getCallback()).isInstanceOf(TelemetryNodeCallback.class); })); } @@ -335,7 +340,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { @Test public void givenOnEveryMessageProcessingSettingsAndSameMessageTwoTimes_whenOnMsg_thenPersistSameMessageTwoTimes() throws TbNodeException { // GIVEN - config.setProcessingSettings(new TbMsgTimeseriesNodeConfiguration.ProcessingSettings.OnEveryMessage()); + config.setProcessingSettings(new OnEveryMessage()); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); @@ -353,7 +358,10 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entityId(msg.getOriginator()) .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) - .strategy(TimeseriesSaveRequest.Strategy.SAVE_ALL) + .strategy(TimeseriesSaveRequest.Strategy.PROCESS_ALL) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) .build(); node.onMsg(ctxMock, msg); @@ -370,7 +378,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { @Test public void givenDeduplicateProcessingSettingsAndSameMessageTwoTimes_whenOnMsg_thenPersistThisMessageOnlyFirstTime() throws TbNodeException { // GIVEN - config.setProcessingSettings(new TbMsgTimeseriesNodeConfiguration.ProcessingSettings.Deduplicate(10)); + config.setProcessingSettings(new Deduplicate(10)); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); @@ -388,7 +396,10 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entityId(msg.getOriginator()) .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) - .strategy(TimeseriesSaveRequest.Strategy.SAVE_ALL) + .strategy(TimeseriesSaveRequest.Strategy.PROCESS_ALL) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) .build(); node.onMsg(ctxMock, msg); @@ -405,7 +416,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { @Test public void givenWebSocketsOnlyProcessingSettingsAndSameMessageTwoTimes_whenOnMsg_thenSendsOnlyWsUpdateTwoTimes() throws TbNodeException { // GIVEN - config.setProcessingSettings(new TbMsgTimeseriesNodeConfiguration.ProcessingSettings.WebSocketsOnly()); + config.setProcessingSettings(new WebSocketsOnly()); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); @@ -424,6 +435,9 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) .strategy(TimeseriesSaveRequest.Strategy.WS_ONLY) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) .build(); node.onMsg(ctxMock, msg); @@ -440,7 +454,8 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { @Test public void givenAdvancedProcessingSettingsWithOnEveryMessageStrategiesForAllActionsAndSameMessageTwoTimes_whenOnMsg_thenPersistSameMessageTwoTimes() throws TbNodeException { // GIVEN - config.setProcessingSettings(new TbMsgTimeseriesNodeConfiguration.ProcessingSettings.Advanced( + config.setProcessingSettings(new Advanced( + ProcessingStrategy.onEveryMessage(), ProcessingStrategy.onEveryMessage(), ProcessingStrategy.onEveryMessage(), ProcessingStrategy.onEveryMessage() @@ -462,7 +477,10 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entityId(msg.getOriginator()) .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) - .strategy(TimeseriesSaveRequest.Strategy.SAVE_ALL) + .strategy(TimeseriesSaveRequest.Strategy.PROCESS_ALL) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) .build(); node.onMsg(ctxMock, msg); @@ -479,10 +497,11 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { @Test public void givenAdvancedProcessingSettingsWithDifferentDeduplicateStrategyForEachAction_whenOnMsg_thenEvaluatesStrategiesForEachActionsIndependently() throws TbNodeException { // GIVEN - config.setProcessingSettings(new TbMsgTimeseriesNodeConfiguration.ProcessingSettings.Advanced( + config.setProcessingSettings(new Advanced( ProcessingStrategy.deduplicate(1), ProcessingStrategy.deduplicate(2), - ProcessingStrategy.deduplicate(3) + ProcessingStrategy.deduplicate(3), + ProcessingStrategy.deduplicate(4) )); node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); @@ -490,6 +509,8 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { long ts1 = 500L; long ts2 = 1500L; long ts3 = 2500L; + long ts4 = 3500L; + long ts5 = 4500L; // WHEN-THEN node.onMsg(ctxMock, TbMsg.newMsg() @@ -499,7 +520,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts1)))) .build()); then(telemetryServiceMock).should().saveTimeseries(assertArg( - actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL) + actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.PROCESS_ALL) )); clearInvocations(telemetryServiceMock); @@ -511,7 +532,9 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts2)))) .build()); then(telemetryServiceMock).should().saveTimeseries(assertArg( - actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo(new TimeseriesSaveRequest.Strategy(true, false, false)) + actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo( + new TimeseriesSaveRequest.Strategy(true, false, false, false) + ) )); clearInvocations(telemetryServiceMock); @@ -523,14 +546,45 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts3)))) .build()); then(telemetryServiceMock).should().saveTimeseries(assertArg( - actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo(new TimeseriesSaveRequest.Strategy(true, true, false)) + actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo( + new TimeseriesSaveRequest.Strategy(true, true, false, false) + ) + )); + + clearInvocations(telemetryServiceMock); + + node.onMsg(ctxMock, TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(DEVICE_ID) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts4)))) + .build()); + then(telemetryServiceMock).should().saveTimeseries(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo( + new TimeseriesSaveRequest.Strategy(true, false, true, false) + ) + )); + + clearInvocations(telemetryServiceMock); + + node.onMsg(ctxMock, TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(DEVICE_ID) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts5)))) + .build()); + then(telemetryServiceMock).should().saveTimeseries(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo( + new TimeseriesSaveRequest.Strategy(true, true, false, true) + ) )); } @Test public void givenAdvancedProcessingSettingsWithSkipStrategiesForAllActionsAndSameMessageTwoTimes_whenOnMsg_thenSkipsSameMessageTwoTimes() throws TbNodeException { // GIVEN - config.setProcessingSettings(new TbMsgTimeseriesNodeConfiguration.ProcessingSettings.Advanced( + config.setProcessingSettings(new Advanced( + ProcessingStrategy.skip(), ProcessingStrategy.skip(), ProcessingStrategy.skip(), ProcessingStrategy.skip() @@ -631,6 +685,9 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { }, "webSockets": { "type": "ON_EVERY_MESSAGE" + }, + "calculatedFields": { + "type": "ON_EVERY_MESSAGE" } } }""") diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java index 8bcb8503d8..38417c3922 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java @@ -43,6 +43,8 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.domain.Domain; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -67,6 +69,7 @@ import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceService; @@ -155,6 +158,8 @@ public class TenantIdLoaderTest { private MobileAppService mobileAppService; @Mock private MobileAppBundleService mobileAppBundleService; + @Mock + private CalculatedFieldService calculatedFieldService; private TenantId tenantId; private TenantProfileId tenantProfileId; @@ -402,6 +407,18 @@ public class TenantIdLoaderTest { when(ctx.getMobileAppBundleService()).thenReturn(mobileAppBundleService); doReturn(mobileAppBundle).when(mobileAppBundleService).findMobileAppBundleById(eq(tenantId), any()); break; + case CALCULATED_FIELD: + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + when(ctx.getCalculatedFieldService()).thenReturn(calculatedFieldService); + doReturn(calculatedField).when(calculatedFieldService).findById(eq(tenantId), any()); + break; + case CALCULATED_FIELD_LINK: + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); + calculatedFieldLink.setTenantId(tenantId); + when(ctx.getCalculatedFieldService()).thenReturn(calculatedFieldService); + doReturn(calculatedFieldLink).when(calculatedFieldService).findCalculatedFieldLinkById(eq(tenantId), any()); + break; default: throw new RuntimeException("Unexpected originator EntityType " + entityType); } diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 5740f199f8..f8df4bb55e 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -133,8 +133,6 @@ redis: transport: # Local CoAP transport parameters coap: - # Enable/disable coap transport protocol. - enabled: "${COAP_ENABLED:true}" # CoaP processing timeout in milliseconds timeout: "${COAP_TIMEOUT:10000}" # CoaP piggyback response timeout in milliseconds @@ -173,8 +171,6 @@ transport: # CoAP server parameters coap: - # Enable/disable coap transport protocol. - enabled: "${COAP_SERVER_ENABLED:true}" # CoAP bind-address bind_address: "${COAP_BIND_ADDRESS:0.0.0.0}" # CoAP bind port @@ -351,6 +347,8 @@ queue: core: # Default topic name topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + # For high-priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_CORE_NOTIFICATIONS_TOPIC:tb_core.notifications}" # Interval in milliseconds to poll messages by Core microservices poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" # Amount of partitions used by Core microservices @@ -381,6 +379,8 @@ queue: rule-engine: # Deprecated. It will be removed in the nearest releases topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" + # For high-priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_RULE_ENGINE_NOTIFICATIONS_TOPIC:tb_rule_engine.notifications}" # Interval in milliseconds to poll messages by Rule Engine poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" # Timeout for processing a message pack of Rule Engine diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index eca40b19fb..d282b50ff3 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -296,6 +296,8 @@ queue: core: # Default topic name topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + # For high-priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_CORE_NOTIFICATIONS_TOPIC:tb_core.notifications}" # Interval in milliseconds to poll messages by Core microservices poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" # Amount of partitions used by Core microservices @@ -326,6 +328,8 @@ queue: rule-engine: # Deprecated. It will be removed in the nearest releases topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" + # For high-priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_RULE_ENGINE_NOTIFICATIONS_TOPIC:tb_rule_engine.notifications}" # Interval in milliseconds to poll messages by Rule Engine poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" # Timeout for processing a message pack of Rule Engine diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index 149fc2a6e2..a198613e11 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -397,6 +397,8 @@ queue: core: # Default topic name topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + # For high-priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_CORE_NOTIFICATIONS_TOPIC:tb_core.notifications}" # Interval in milliseconds to poll messages by Core microservices poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" # Amount of partitions used by Core microservices @@ -427,6 +429,8 @@ queue: rule-engine: # Deprecated. It will be removed in the nearest releases topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" + # For high-priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_RULE_ENGINE_NOTIFICATIONS_TOPIC:tb_rule_engine.notifications}" # Interval in milliseconds to poll messages by Rule Engine poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" # Timeout for processing a message pack of Rule Engine diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index b35ce5e7be..fb75203499 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -330,6 +330,8 @@ queue: core: # Default topic name topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + # For high-priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_CORE_NOTIFICATIONS_TOPIC:tb_core.notifications}" # Interval in milliseconds to poll messages by Core microservices poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" # Amount of partitions used by Core microservices @@ -360,6 +362,8 @@ queue: rule-engine: # Deprecated. It will be removed in the nearest releases topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" + # For high-priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_RULE_ENGINE_NOTIFICATIONS_TOPIC:tb_rule_engine.notifications}" # Interval in milliseconds to poll messages by Rule Engine poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" # Timeout for processing a message pack of Rule Engine diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index 4de8a0e2c5..281e221674 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -283,6 +283,8 @@ queue: core: # Default topic name topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + # For high-priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_CORE_NOTIFICATIONS_TOPIC:tb_core.notifications}" # Interval in milliseconds to poll messages by Core microservices poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" # Amount of partitions used by Core microservices @@ -313,6 +315,8 @@ queue: rule-engine: # Deprecated. It will be removed in the nearest releases topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" + # For high-priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_RULE_ENGINE_NOTIFICATIONS_TOPIC:tb_rule_engine.notifications}" # Interval in milliseconds to poll messages by Rule Engine poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" # Timeout for processing a message pack of Rule Engine diff --git a/ui-ngx/src/app/core/auth/auth.models.ts b/ui-ngx/src/app/core/auth/auth.models.ts index 55af791156..1944c693ac 100644 --- a/ui-ngx/src/app/core/auth/auth.models.ts +++ b/ui-ngx/src/app/core/auth/auth.models.ts @@ -28,6 +28,8 @@ export interface SysParamsState { userSettings: UserSettings; maxResourceSize: number; maxDebugModeDurationMinutes: number; + maxDataPointsPerRollingArg: number; + maxArgumentsPerCF: number; ruleChainDebugPerTenantLimitsConfiguration?: string; } diff --git a/ui-ngx/src/app/core/auth/auth.reducer.ts b/ui-ngx/src/app/core/auth/auth.reducer.ts index a460cf35bb..3ecf70074c 100644 --- a/ui-ngx/src/app/core/auth/auth.reducer.ts +++ b/ui-ngx/src/app/core/auth/auth.reducer.ts @@ -31,6 +31,8 @@ const emptyUserAuthState: AuthPayload = { persistDeviceStateToTelemetry: false, mobileQrEnabled: false, maxResourceSize: 0, + maxArgumentsPerCF: 0, + maxDataPointsPerRollingArg: 0, maxDebugModeDurationMinutes: 0, userSettings: initialUserSettings }; diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts new file mode 100644 index 0000000000..66c0cb609e --- /dev/null +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -0,0 +1,61 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Injectable } from '@angular/core'; +import { defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; +import { Observable } from 'rxjs'; +import { HttpClient } from '@angular/common/http'; +import { PageData } from '@shared/models/page/page-data'; +import { CalculatedField, CalculatedFieldTestScriptInputParams } from '@shared/models/calculated-field.models'; +import { PageLink } from '@shared/models/page/page-link'; +import { EntityId } from '@shared/models/id/entity-id'; +import { EntityTestScriptResult } from '@shared/models/entity.models'; +import { CalculatedFieldEventBody } from '@shared/models/event.models'; + +@Injectable({ + providedIn: 'root' +}) +export class CalculatedFieldsService { + + constructor( + private http: HttpClient + ) { } + + public getCalculatedFieldById(calculatedFieldId: string, config?: RequestConfig): Observable { + return this.http.get(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); + } + + public saveCalculatedField(calculatedField: CalculatedField, config?: RequestConfig): Observable { + return this.http.post('/api/calculatedField', calculatedField, defaultHttpOptionsFromConfig(config)); + } + + public deleteCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { + return this.http.delete(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); + } + + public getCalculatedFields({ entityType, id }: EntityId, pageLink: PageLink, config?: RequestConfig): Observable> { + return this.http.get>(`/api/${entityType}/${id}/calculatedFields${pageLink.toQuery()}`, + defaultHttpOptionsFromConfig(config)); + } + + public testScript(inputParams: CalculatedFieldTestScriptInputParams, config?: RequestConfig): Observable { + return this.http.post('/api/calculatedField/testScript', inputParams, defaultHttpOptionsFromConfig(config)); + } + + public getLatestCalculatedFieldDebugEvent(id: string, config?: RequestConfig): Observable { + return this.http.get(`/api/calculatedField/${id}/debug`, defaultHttpOptionsFromConfig(config)); + } +} diff --git a/ui-ngx/src/app/core/http/resource.service.ts b/ui-ngx/src/app/core/http/resource.service.ts index 9f0daa73ee..615b721b97 100644 --- a/ui-ngx/src/app/core/http/resource.service.ts +++ b/ui-ngx/src/app/core/http/resource.service.ts @@ -90,8 +90,8 @@ export class ResourceService { return this.http.post('/api/resource', resource, defaultHttpOptionsFromConfig(config)); } - public deleteResource(resourceId: string, config?: RequestConfig) { - return this.http.delete(`/api/resource/${resourceId}`, defaultHttpOptionsFromConfig(config)); + public deleteResource(resourceId: string, force = false, config?: RequestConfig) { + return this.http.delete(`/api/resource/${resourceId}?force=${force}`, defaultHttpOptionsFromConfig(config)); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts new file mode 100644 index 0000000000..cd9f0373db --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -0,0 +1,317 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + DateEntityTableColumn, + EntityTableColumn, + EntityTableConfig +} from '@home/models/entity/entities-table-config.models'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { TranslateService } from '@ngx-translate/core'; +import { Direction } from '@shared/models/page/sort-order'; +import { MatDialog } from '@angular/material/dialog'; +import { PageLink } from '@shared/models/page/page-link'; +import { Observable, of } from 'rxjs'; +import { PageData } from '@shared/models/page/page-data'; +import { EntityId } from '@shared/models/id/entity-id'; +import { MINUTE } from '@shared/models/time/time.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { getCurrentAuthState, getCurrentAuthUser } from '@core/auth/auth.selectors'; +import { DestroyRef, Renderer2 } from '@angular/core'; +import { EntityDebugSettings } from '@shared/models/entity.models'; +import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { EntityDebugSettingsPanelComponent } from '@home/components/entity/debug/entity-debug-settings-panel.component'; +import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; +import { catchError, filter, switchMap, tap } from 'rxjs/operators'; +import { + ArgumentType, + CalculatedField, + CalculatedFieldEventArguments, + CalculatedFieldDebugDialogData, + CalculatedFieldDialogData, + CalculatedFieldTestScriptDialogData, + getCalculatedFieldArgumentsEditorCompleter, + getCalculatedFieldArgumentsHighlights, + CalculatedFieldTypeTranslations, + CalculatedFieldType, +} from '@shared/models/calculated-field.models'; +import { + CalculatedFieldDebugDialogComponent, + CalculatedFieldDialogComponent, + CalculatedFieldScriptTestDialogComponent +} from './components/public-api'; +import { ImportExportService } from '@shared/import-export/import-export.service'; +import { isObject } from '@core/utils'; +import { DatePipe } from '@angular/common'; + +export class CalculatedFieldsTableConfig extends EntityTableConfig { + + // TODO: [Calculated Fields] remove hardcode when BE variable implemented + readonly calculatedFieldsDebugPerTenantLimitsConfiguration = + getCurrentAuthState(this.store)['calculatedFieldsDebugPerTenantLimitsConfiguration'] || '1:1'; + readonly maxDebugModeDuration = getCurrentAuthState(this.store).maxDebugModeDurationMinutes * MINUTE; + readonly tenantId = getCurrentAuthUser(this.store).tenantId; + additionalDebugActionConfig = { + title: this.translate.instant('calculated-fields.see-debug-events'), + action: (calculatedField: CalculatedField) => this.openDebugEventsDialog.call(this, calculatedField), + }; + + constructor(private calculatedFieldsService: CalculatedFieldsService, + private translate: TranslateService, + private dialog: MatDialog, + private datePipe: DatePipe, + public entityId: EntityId = null, + private store: Store, + private durationLeft: DurationLeftPipe, + private popoverService: TbPopoverService, + private destroyRef: DestroyRef, + private renderer: Renderer2, + public entityName: string, + private importExportService: ImportExportService + ) { + super(); + this.tableTitle = this.translate.instant('entity.type-calculated-fields'); + this.detailsPanelEnabled = false; + this.pageMode = false; + this.entityType = EntityType.CALCULATED_FIELD; + this.entityTranslations = entityTypeTranslations.get(EntityType.CALCULATED_FIELD); + + this.entitiesFetchFunction = (pageLink: PageLink) => this.fetchCalculatedFields(pageLink); + this.addEntity = this.getCalculatedFieldDialog.bind(this); + this.deleteEntityTitle = (field: CalculatedField) => this.translate.instant('calculated-fields.delete-title', {title: field.name}); + this.deleteEntityContent = () => this.translate.instant('calculated-fields.delete-text'); + this.deleteEntitiesTitle = count => this.translate.instant('calculated-fields.delete-multiple-title', {count}); + this.deleteEntitiesContent = () => this.translate.instant('calculated-fields.delete-multiple-text'); + this.deleteEntity = id => this.calculatedFieldsService.deleteCalculatedField(id.id); + this.addActionDescriptors = [ + { + name: this.translate.instant('calculated-fields.create'), + icon: 'insert_drive_file', + isEnabled: () => true, + onAction: ($event) => this.getTable().addEntity($event) + }, + { + name: this.translate.instant('calculated-fields.import'), + icon: 'file_upload', + isEnabled: () => true, + onAction: () => this.importCalculatedField() + } + ]; + + this.defaultSortOrder = {property: 'createdTime', direction: Direction.DESC}; + + const expressionColumn = new EntityTableColumn('expression', 'calculated-fields.expression', '300px'); + expressionColumn.sortable = false; + expressionColumn.cellContentFunction = entity => { + const expressionLabel = this.getExpressionLabel(entity); + return expressionLabel.length < 45 ? expressionLabel : `${expressionLabel.substring(0, 44)}…`; + } + expressionColumn.cellTooltipFunction = entity => { + const expressionLabel = this.getExpressionLabel(entity); + return expressionLabel.length < 45 ? null : expressionLabel + }; + + this.columns.push(new DateEntityTableColumn('createdTime', 'common.created-time', this.datePipe, '150px')); + this.columns.push(new EntityTableColumn('name', 'common.name', '33%')); + this.columns.push(new EntityTableColumn('type', 'common.type', '50px', entity => this.translate.instant(CalculatedFieldTypeTranslations.get(entity.type)))); + this.columns.push(expressionColumn); + + this.cellActionDescriptors.push( + { + name: this.translate.instant('action.export'), + icon: 'file_download', + isEnabled: () => true, + onAction: (event$, entity) => this.exportCalculatedField(event$, entity), + }, + { + name: this.translate.instant('entity-view.events'), + icon: 'mdi:clipboard-text-clock', + isEnabled: () => true, + onAction: (_, entity) => this.openDebugEventsDialog(entity), + }, + { + name: '', + nameFunction: entity => this.getDebugConfigLabel(entity?.debugSettings), + icon: 'mdi:bug', + isEnabled: () => true, + iconFunction: ({ debugSettings }) => this.isDebugActive(debugSettings?.allEnabledUntil) || debugSettings?.failuresEnabled ? 'mdi:bug' : 'mdi:bug-outline', + onAction: ($event, entity) => this.onOpenDebugConfig($event, entity), + }, + { + name: this.translate.instant('action.edit'), + icon: 'edit', + isEnabled: () => true, + onAction: (_, entity) => this.editCalculatedField(entity), + } + ); + } + + private getExpressionLabel(entity: CalculatedField): string { + if (entity.type === CalculatedFieldType.SCRIPT) { + return 'function calculate(' + Object.keys(entity.configuration.arguments).join(', ') + ')'; + } else { + return entity.configuration.expression; + } + } + + fetchCalculatedFields(pageLink: PageLink): Observable> { + return this.calculatedFieldsService.getCalculatedFields(this.entityId, pageLink); + } + + onOpenDebugConfig($event: Event, calculatedField: CalculatedField): void { + const { debugSettings = {}, id } = calculatedField; + const additionalActionConfig = { + ...this.additionalDebugActionConfig, + action: () => this.openDebugEventsDialog(calculatedField) + }; + const { viewContainerRef } = this.getTable(); + if ($event) { + $event.stopPropagation(); + } + const trigger = $event.target as Element; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const debugStrategyPopover = this.popoverService.displayPopover(trigger, this.renderer, + viewContainerRef, EntityDebugSettingsPanelComponent, 'bottom', true, null, + { + debugLimitsConfiguration: this.calculatedFieldsDebugPerTenantLimitsConfiguration, + maxDebugModeDuration: this.maxDebugModeDuration, + entityLabel: this.translate.instant('debug-settings.calculated-field'), + additionalActionConfig, + ...debugSettings + }, + {}, + {}, {}, true); + debugStrategyPopover.tbComponentRef.instance.onSettingsApplied.pipe(takeUntilDestroyed(this.destroyRef)).subscribe((settings: EntityDebugSettings) => { + this.onDebugConfigChanged(id.id, settings); + debugStrategyPopover.hide(); + }); + } + } + + private editCalculatedField(calculatedField: CalculatedField, isDirty = false): void { + this.getCalculatedFieldDialog(calculatedField, 'action.apply', isDirty) + .subscribe((res) => { + if (res) { + this.updateData(); + } + }); + } + + private getCalculatedFieldDialog(value?: CalculatedField, buttonTitle = 'action.add', isDirty = false): Observable { + return this.dialog.open(CalculatedFieldDialogComponent, { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], + data: { + value, + buttonTitle, + entityId: this.entityId, + debugLimitsConfiguration: this.calculatedFieldsDebugPerTenantLimitsConfiguration, + tenantId: this.tenantId, + entityName: this.entityName, + additionalDebugActionConfig: this.additionalDebugActionConfig, + getTestScriptDialogFn: this.getTestScriptDialog.bind(this), + isDirty, + }, + enterAnimationDuration: isDirty ? 0 : null, + }) + .afterClosed() + .pipe(filter(Boolean)); + } + + private openDebugEventsDialog(calculatedField: CalculatedField): void { + this.dialog.open(CalculatedFieldDebugDialogComponent, { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], + data: { + tenantId: this.tenantId, + value: calculatedField, + getTestScriptDialogFn: this.getTestScriptDialog.bind(this), + } + }) + .afterClosed() + .subscribe(); + } + + private exportCalculatedField($event: Event, calculatedField: CalculatedField): void { + if ($event) { + $event.stopPropagation(); + } + this.importExportService.exportCalculatedField(calculatedField.id.id); + } + + private importCalculatedField(): void { + this.importExportService.importCalculatedField(this.entityId) + .pipe(filter(Boolean), takeUntilDestroyed(this.destroyRef)) + .subscribe(() => this.updateData()); + } + + private getDebugConfigLabel(debugSettings: EntityDebugSettings): string { + const isDebugActive = this.isDebugActive(debugSettings?.allEnabledUntil); + + if (!isDebugActive) { + return debugSettings?.failuresEnabled ? this.translate.instant('debug-settings.failures') : this.translate.instant('common.disabled'); + } else { + return this.durationLeft.transform(debugSettings?.allEnabledUntil); + } + } + + private isDebugActive(allEnabledUntil: number): boolean { + return allEnabledUntil > new Date().getTime(); + } + + private onDebugConfigChanged(id: string, debugSettings: EntityDebugSettings): void { + this.calculatedFieldsService.getCalculatedFieldById(id).pipe( + switchMap(field => this.calculatedFieldsService.saveCalculatedField({ ...field, debugSettings })), + catchError(() => of(null)), + takeUntilDestroyed(this.destroyRef), + ).subscribe(() => this.updateData()); + } + + private getTestScriptDialog(calculatedField: CalculatedField, argumentsObj?: CalculatedFieldEventArguments, openCalculatedFieldEdit = true): Observable { + const resultArguments = Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { + const type = calculatedField.configuration.arguments[key].refEntityKey.type; + acc[key] = isObject(argumentsObj) && argumentsObj.hasOwnProperty(key) + ? { ...argumentsObj[key], type } + : type === ArgumentType.Rolling ? { values: [], type } : { value: '', type, ts: new Date().getTime() }; + return acc; + }, {}); + return this.dialog.open(CalculatedFieldScriptTestDialogComponent, + { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog', 'tb-fullscreen-dialog-gt-xs'], + data: { + arguments: resultArguments, + expression: calculatedField.configuration.expression, + argumentsEditorCompleter: getCalculatedFieldArgumentsEditorCompleter(calculatedField.configuration.arguments), + argumentsHighlightRules: getCalculatedFieldArgumentsHighlights(calculatedField.configuration.arguments), + openCalculatedFieldEdit + } + }).afterClosed() + .pipe( + filter(Boolean), + tap(expression => { + if (openCalculatedFieldEdit) { + this.editCalculatedField({ entityId: this.entityId, ...calculatedField, configuration: {...calculatedField.configuration, expression } }, true) + } + }), + ); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html new file mode 100644 index 0000000000..df433bc70e --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html @@ -0,0 +1,20 @@ + +@if (calculatedFieldsTableConfig) { + +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.scss new file mode 100644 index 0000000000..3feb1e7429 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.scss @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host ::ng-deep { + tb-entities-table { + .mat-drawer-container { + background-color: white; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts new file mode 100644 index 0000000000..1d0b9dcb15 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -0,0 +1,88 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + ChangeDetectionStrategy, + ChangeDetectorRef, + Component, + DestroyRef, + effect, + input, + Renderer2, + ViewChild, +} from '@angular/core'; +import { EntityId } from '@shared/models/id/entity-id'; +import { EntitiesTableComponent } from '@home/components/entity/entities-table.component'; +import { TranslateService } from '@ngx-translate/core'; +import { MatDialog } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { CalculatedFieldsTableConfig } from '@home/components/calculated-fields/calculated-fields-table-config'; +import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; +import { ImportExportService } from '@shared/import-export/import-export.service'; +import { DatePipe } from '@angular/common'; + +@Component({ + selector: 'tb-calculated-fields-table', + templateUrl: './calculated-fields-table.component.html', + styleUrls: ['./calculated-fields-table.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class CalculatedFieldsTableComponent { + + @ViewChild(EntitiesTableComponent, {static: true}) entitiesTable: EntitiesTableComponent; + + active = input(); + entityId = input(); + entityName = input(); + + calculatedFieldsTableConfig: CalculatedFieldsTableConfig; + + constructor(private calculatedFieldsService: CalculatedFieldsService, + private translate: TranslateService, + private dialog: MatDialog, + private store: Store, + private datePipe: DatePipe, + private durationLeft: DurationLeftPipe, + private popoverService: TbPopoverService, + private cd: ChangeDetectorRef, + private renderer: Renderer2, + private importExportService: ImportExportService, + private destroyRef: DestroyRef) { + + effect(() => { + if (this.active()) { + this.calculatedFieldsTableConfig = new CalculatedFieldsTableConfig( + this.calculatedFieldsService, + this.translate, + this.dialog, + this.datePipe, + this.entityId(), + this.store, + this.durationLeft, + this.popoverService, + this.destroyRef, + this.renderer, + this.entityName(), + this.importExportService + ); + this.cd.markForCheck(); + } + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html new file mode 100644 index 0000000000..162bd3aa1e --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -0,0 +1,149 @@ + +
    +
    + + + +
    {{ 'common.name' | translate }}
    +     + +
    +
    {{ argument.argumentName }}
    + +
    +     +     + + + {{ 'entity.entity-type' | translate }} + + +
    + @if (argument.refEntityId?.entityType === ArgumentEntityType.Tenant) { + {{ 'calculated-fields.argument-current-tenant' | translate }} + } @else if (argument.refEntityId?.id) { + {{ entityTypeTranslations.get(argument.refEntityId.entityType).type | translate }} + } @else { + {{ 'calculated-fields.argument-current' | translate }} + } +
    +     +     + + + {{ 'entity-view.target-entity' | translate }} + + +
    + @if (argument.refEntityId?.id) { + + {{ entityNameMap.get(argument.refEntityId.id) ?? '' }} + + } +
    +     +     + + + {{ 'common.type' | translate }} + + +
    {{ ArgumentTypeTranslations.get(argument.refEntityKey.type) | translate }}
    +     +     + + + {{ 'entity.key' | translate }} + + + +
    {{ argument.refEntityKey.key }}
    +     +     +     + + + +
    + + +
    +     +     + + +
    +
    + {{ 'calculated-fields.no-arguments' | translate }} +
    + @if (errorText) { + + } +
    +
    + + @if (maxArgumentsPerCF && argumentsFormArray.length >= maxArgumentsPerCF) { +
    + warning + {{ 'calculated-fields.hint.max-args' | translate }} +
    + } +
    +
    diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss new file mode 100644 index 0000000000..ae8fd25170 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -0,0 +1,82 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .arguments-table { + min-height: 108px; + + &-with-error { + min-height: 150px; + } + + .mat-mdc-table { + table-layout: fixed; + } + + .key-text { + font-size: 13px; + } + + .copy-argument-name { + visibility: hidden; + transition: visibility 0.1s; + } + + .argument-name-cell:hover { + .copy-argument-name { + visibility: visible; + } + } + } + + .max-args-warning { + .mat-icon { + color: #FAA405; + } + } + + .tb-form-table-row-cell-buttons { + --mat-badge-legacy-small-size-container-size: 8px; + --mat-badge-small-size-container-overlap-offset: -5px; + --mat-badge-small-size-text-size: 0; + } +} + +:host ::ng-deep { + .mat-mdc-standard-chip { + .mdc-evolution-chip__cell--primary, .mdc-evolution-chip__action--primary, .mdc-evolution-chip__text-label { + overflow: hidden; + } + } + + .arguments-table:not(.arguments-table-with-error) { + .mdc-data-table__row:last-child .mat-mdc-cell { + border-bottom: none; + } + } + + .arguments-table { + .mat-mdc-header-row.mat-row-select .mat-mdc-header-cell.entity-type-header { + padding: 0 28px 0 0; + } + } + + .copy-argument-name { + .mat-icon { + font-size: 16px; + padding: 4px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts new file mode 100644 index 0000000000..d1ee0093b1 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -0,0 +1,279 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + AfterViewInit, + ChangeDetectorRef, + Component, + DestroyRef, + forwardRef, + Input, + OnChanges, + Renderer2, + SimpleChanges, + ViewChild, + ViewContainerRef, +} from '@angular/core'; +import { + AbstractControl, + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, +} from '@angular/forms'; +import { + ArgumentEntityType, + ArgumentType, + ArgumentTypeTranslations, + CalculatedFieldArgument, + CalculatedFieldArgumentValue, + CalculatedFieldType, +} from '@shared/models/calculated-field.models'; +import { CalculatedFieldArgumentPanelComponent } from '@home/components/calculated-fields/components/public-api'; +import { MatButton } from '@angular/material/button'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { EntityId } from '@shared/models/id/entity-id'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { getEntityDetailsPageURL, isDefined, isDefinedAndNotNull, isEqual } from '@core/utils'; +import { TbPopoverComponent } from '@shared/components/popover.component'; +import { TbTableDatasource } from '@shared/components/table/table-datasource.abstract'; +import { EntityService } from '@core/http/entity.service'; +import { MatSort } from '@angular/material/sort'; +import { getCurrentAuthState } from '@core/auth/auth.selectors'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; + +@Component({ + selector: 'tb-calculated-field-arguments-table', + templateUrl: './calculated-field-arguments-table.component.html', + styleUrls: [`calculated-field-arguments-table.component.scss`], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => CalculatedFieldArgumentsTableComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => CalculatedFieldArgumentsTableComponent), + multi: true + } + ], +}) +export class CalculatedFieldArgumentsTableComponent implements ControlValueAccessor, Validator, OnChanges, AfterViewInit { + + @Input() entityId: EntityId; + @Input() tenantId: string; + @Input() entityName: string; + @Input() calculatedFieldType: CalculatedFieldType; + + @ViewChild(MatSort, { static: true }) sort: MatSort; + + errorText = ''; + argumentsFormArray = this.fb.array([]); + entityNameMap = new Map(); + sortOrder = { direction: 'asc', property: '' }; + dataSource = new CalculatedFieldArgumentDatasource(); + + readonly entityTypeTranslations = entityTypeTranslations; + readonly ArgumentTypeTranslations = ArgumentTypeTranslations; + readonly ArgumentEntityType = ArgumentEntityType; + readonly ArgumentType = ArgumentType; + readonly CalculatedFieldType = CalculatedFieldType; + readonly maxArgumentsPerCF = getCurrentAuthState(this.store).maxArgumentsPerCF; + + private popoverComponent: TbPopoverComponent; + private propagateChange: (argumentsObj: Record) => void = () => {}; + + constructor( + private fb: FormBuilder, + private popoverService: TbPopoverService, + private viewContainerRef: ViewContainerRef, + private cd: ChangeDetectorRef, + private renderer: Renderer2, + private entityService: EntityService, + private destroyRef: DestroyRef, + private store: Store + ) { + this.argumentsFormArray.valueChanges.pipe(takeUntilDestroyed()).subscribe(value => { + this.updateEntityNameMap(value); + this.updateDataSource(value); + this.propagateChange(this.getArgumentsObject(value)); + }); + } + + ngOnChanges(changes: SimpleChanges): void { + if (changes.calculatedFieldType?.previousValue + && changes.calculatedFieldType.currentValue !== changes.calculatedFieldType.previousValue) { + this.argumentsFormArray.updateValueAndValidity(); + } + } + + ngAfterViewInit(): void { + this.sort.sortChange.asObservable().pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => { + this.sortOrder.property = this.sort.active; + this.sortOrder.direction = this.sort.direction; + this.updateDataSource(this.argumentsFormArray.value); + }); + } + + registerOnChange(fn: (argumentsObj: Record) => void): void { + this.propagateChange = fn; + } + + registerOnTouched(_): void {} + + validate(): ValidationErrors | null { + this.updateErrorText(); + return this.errorText ? { argumentsFormArray: false } : null; + } + + onDelete($event: Event, argument: CalculatedFieldArgumentValue): void { + $event.stopPropagation(); + const index = this.argumentsFormArray.controls.findIndex(control => isEqual(control.value, argument)); + this.argumentsFormArray.removeAt(index); + this.argumentsFormArray.markAsDirty(); + } + + manageArgument($event: Event, matButton: MatButton, argument = {} as CalculatedFieldArgumentValue, index?: number): void { + $event?.stopPropagation(); + if (this.popoverComponent && !this.popoverComponent.tbHidden) { + this.popoverComponent.hide(); + } + const trigger = matButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const ctx = { + index, + argument, + entityId: this.entityId, + calculatedFieldType: this.calculatedFieldType, + buttonTitle: this.argumentsFormArray.at(index)?.value ? 'action.apply' : 'action.add', + tenantId: this.tenantId, + entityName: this.entityName, + usedArgumentNames: this.argumentsFormArray.value.map(({ argumentName }) => argumentName).filter(name => name !== argument.argumentName), + }; + this.popoverComponent = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, CalculatedFieldArgumentPanelComponent, isDefined(index) ? 'left' : 'right', false, null, + ctx, + {}, + {}, {}, true); + this.popoverComponent.tbComponentRef.instance.argumentsDataApplied.subscribe(({ value, index }) => { + this.popoverComponent.hide(); + const formGroup = this.fb.group(value); + if (isDefinedAndNotNull(index)) { + this.argumentsFormArray.setControl(index, formGroup); + } else { + this.argumentsFormArray.push(formGroup); + } + formGroup.markAsDirty(); + this.cd.markForCheck(); + }); + } + } + + private updateDataSource(value: CalculatedFieldArgumentValue[]): void { + const sortedValue = this.sortData(value); + this.dataSource.loadData(sortedValue); + } + + private updateErrorText(): void { + if (this.calculatedFieldType === CalculatedFieldType.SIMPLE + && this.argumentsFormArray.controls.some(control => control.value.refEntityKey.type === ArgumentType.Rolling)) { + this.errorText = 'calculated-fields.hint.arguments-simple-with-rolling'; + } else if (!this.argumentsFormArray.controls.length) { + this.errorText = 'calculated-fields.hint.arguments-empty'; + } else { + this.errorText = ''; + } + } + + private getArgumentsObject(value: CalculatedFieldArgumentValue[]): Record { + return value.reduce((acc, argumentValue) => { + const { argumentName, ...argument } = argumentValue as CalculatedFieldArgumentValue; + acc[argumentName] = argument; + return acc; + }, {} as Record); + } + + writeValue(argumentsObj: Record): void { + this.argumentsFormArray.clear(); + this.populateArgumentsFormArray(argumentsObj) + } + + getEntityDetailsPageURL(id: string, type: EntityType): string { + return getEntityDetailsPageURL(id, type); + } + + private populateArgumentsFormArray(argumentsObj: Record): void { + Object.keys(argumentsObj).forEach(key => { + const value: CalculatedFieldArgumentValue = { + ...argumentsObj[key], + argumentName: key + }; + this.argumentsFormArray.push(this.fb.group(value), { emitEvent: false }); + }); + this.argumentsFormArray.updateValueAndValidity(); + } + + private updateEntityNameMap(value: CalculatedFieldArgumentValue[]): void { + value.forEach(({ refEntityId = {}}) => { + if (refEntityId.id && !this.entityNameMap.has(refEntityId.id)) { + const { id, entityType } = refEntityId as EntityId; + this.entityService.getEntity(entityType as EntityType, id, { ignoreLoading: true, ignoreErrors: true }) + .pipe(takeUntilDestroyed(this.destroyRef)) + .subscribe(entity => this.entityNameMap.set(id, entity.name)); + } + }); + } + + private getSortValue(argument: CalculatedFieldArgumentValue, column: string): string { + switch (column) { + case 'entityType': + if (argument.refEntityId?.entityType === ArgumentEntityType.Tenant) { + return 'calculated-fields.argument-current-tenant'; + } else if (argument.refEntityId?.id) { + return entityTypeTranslations.get((argument.refEntityId)?.entityType as unknown as EntityType).type; + } else { + return 'calculated-fields.argument-current'; + } + case 'type': + return ArgumentTypeTranslations.get(argument.refEntityKey.type); + case 'key': + return argument.refEntityKey.key; + default: + return argument.argumentName; + } + } + + private sortData(data: CalculatedFieldArgumentValue[]): CalculatedFieldArgumentValue[] { + return data.sort((a, b) => { + const valA = this.getSortValue(a, this.sortOrder.property) ?? ''; + const valB = this.getSortValue(b, this.sortOrder.property) ?? ''; + return (this.sortOrder.direction === 'asc' ? 1 : -1) * valA.localeCompare(valB); + }); + } +} + +class CalculatedFieldArgumentDatasource extends TbTableDatasource { + constructor() { + super(); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html new file mode 100644 index 0000000000..d9cb7eb021 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -0,0 +1,48 @@ + +
    + +

    {{ 'calculated-fields.debugging' | translate}}

    + + +     +
    + +
    +
    + +
    +
    diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss new file mode 100644 index 0000000000..c4798d06f9 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .debug-dialog-container { + width: 1080px; + max-width: 100%; + + .debug-dialog-content { + height: 65vh; + border-radius: 0; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts new file mode 100644 index 0000000000..8618a11990 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts @@ -0,0 +1,60 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { AfterViewInit, Component, Inject, ViewChild } from '@angular/core'; +import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { Router } from '@angular/router'; +import { DialogComponent } from '@shared/components/dialog.component'; +import { CalculatedFieldEventBody, DebugEventType, EventType } from '@shared/models/event.models'; +import { EventTableComponent } from '@home/components/event/event-table.component'; +import { CalculatedFieldDebugDialogData, CalculatedFieldType } from '@shared/models/calculated-field.models'; + +@Component({ + selector: 'tb-calculated-field-debug-dialog', + styleUrls: ['calculated-field-debug-dialog.component.scss'], + templateUrl: './calculated-field-debug-dialog.component.html', +}) +export class CalculatedFieldDebugDialogComponent extends DialogComponent implements AfterViewInit { + + @ViewChild(EventTableComponent, {static: true}) eventsTable: EventTableComponent; + + readonly DebugEventType = DebugEventType; + readonly debugEventTypes = DebugEventType; + readonly EventType = EventType; + + constructor(protected store: Store, + protected router: Router, + @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldDebugDialogData, + protected dialogRef: MatDialogRef) { + super(store, router, dialogRef); + } + + ngAfterViewInit(): void { + this.eventsTable.entitiesTable.updateData(); + this.eventsTable.entitiesTable.cellActionDescriptors[0].isEnabled = () => this.data.value.type === CalculatedFieldType.SCRIPT; + } + + cancel(): void { + this.dialogRef.close(null); + } + + onDebugEventSelected(event: CalculatedFieldEventBody): void { + this.data.getTestScriptDialogFn(this.data.value, JSON.parse(event.arguments)) + .subscribe(expression => this.dialogRef.close(expression)); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html new file mode 100644 index 0000000000..471c722145 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -0,0 +1,202 @@ + +
    + +

    {{ 'entity.type-calculated-field' | translate}}

    + +
    + +     +
    +
    +
    +
    {{ 'common.general' | translate }}
    +
    + + {{ 'entity-field.title' | translate }} + + @if (fieldFormGroup.get('name').errors && fieldFormGroup.get('name').touched) { + + @if (fieldFormGroup.get('name').hasError('required')) { + {{ 'common.hint.title-required' | translate }} + } @else if (fieldFormGroup.get('name').hasError('pattern')) { + {{ 'common.hint.title-pattern' | translate }} + } @else if (fieldFormGroup.get('name').hasError('maxlength')) { + {{ 'common.hint.title-max-length' | translate }} + } + + } + + +
    + + {{ 'common.type' | translate }} + + @for (type of fieldTypes; track type) { + {{ CalculatedFieldTypeTranslations.get(type) | translate}} + } + + +
    + +
    +
    {{ 'calculated-fields.arguments' | translate }}
    + +
    +
    +
    {{ 'calculated-fields.expression' | translate }}
    + + + @if (configFormGroup.get('expressionSIMPLE').errors && configFormGroup.get('expressionSIMPLE').touched) { + + @if (configFormGroup.get('expressionSIMPLE').hasError('required')) { + {{ 'calculated-fields.hint.expression-required' | translate }} + } @else if (configFormGroup.get('expressionSIMPLE').hasError('pattern')) { + {{ 'calculated-fields.hint.expression-invalid' | translate }} + } @else if (configFormGroup.get('expressionSIMPLE').hasError('maxLength')) { + {{ 'calculated-fields.hint.expression-max-length' | translate }} + } + + } @else { + {{ 'calculated-fields.hint.expression' | translate }} + } + +
    + +
    {{ 'api-usage.tbel' | translate }}
    + +     +
    + +
    +
    +
    +
    +
    {{ 'calculated-fields.output' | translate }}
    +
    + + {{ 'calculated-fields.output-type' | translate }} + + @for (type of outputTypes; track type) { + {{ OutputTypeTranslations.get(type) | translate}} + } + + + @if (outputFormGroup.get('type').value === OutputType.Attribute + && (data.entityId.entityType === EntityType.DEVICE || data.entityId.entityType === EntityType.DEVICE_PROFILE)) { + + {{ 'calculated-fields.attribute-scope' | translate }} + + + {{ 'calculated-fields.server-attributes' | translate }} + + + {{ 'calculated-fields.shared-attributes' | translate }} + + + + } +
    + @if (fieldFormGroup.get('type').value === CalculatedFieldType.SIMPLE) { +
    + + + {{ (outputFormGroup.get('type').value === OutputType.Timeseries + ? 'calculated-fields.timeseries-key' + : 'calculated-fields.attribute-key') + | translate }} + + + @if (outputFormGroup.get('name').errors && outputFormGroup.get('name').touched) { + + @if (outputFormGroup.get('name').hasError('required')) { + {{ 'common.hint.key-required' | translate }} + } @else if (outputFormGroup.get('name').hasError('pattern')) { + {{ 'common.hint.key-pattern' | translate }} + } @else if (outputFormGroup.get('name').hasError('maxlength')) { + {{ 'common.hint.key-max-length' | translate }} + } + + } + + + {{ 'calculated-fields.decimals-by-default' | translate }} + + @if (outputFormGroup.get('decimalsByDefault').errors && outputFormGroup.get('decimalsByDefault').touched) { + {{ 'calculated-fields.hint.decimals-range' | translate }} + } + +
    + } +
    +     +
    +
    +
    + + +
    +
    diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss new file mode 100644 index 0000000000..8bc422eed1 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.calculated-field-dialog-container { + width: 869px; + max-width: 100%; +} + +.tbel-script-lang-chip { + line-height: 20px; + font-size: 14px; + font-weight: 500; + color: white; + border-radius: 100px; + width: 70px; + min-width: 70px; + display: flex; + justify-content: center; + margin-top: 2px; + margin-right: 4px; +} + +.tb-js-func { + .ace_tb { + &.ace_calculated-field { + &-ctx { + color: #C52F00; + } + &-args { + color: #185F2A; + } + &-key { + color: #c24c1a; + } + &-time-window, &-values, &-func, &-value, &-ts { + color: #7214D0; + } + &-start-ts, &-end-ts { + color: #2CAA00; + } + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts new file mode 100644 index 0000000000..052b660c3c --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -0,0 +1,223 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, DestroyRef, Inject, ViewEncapsulation } from '@angular/core'; +import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { Router } from '@angular/router'; +import { DialogComponent } from '@shared/components/dialog.component'; +import { + CalculatedField, + CalculatedFieldConfiguration, + calculatedFieldDefaultScript, + CalculatedFieldDialogData, + CalculatedFieldType, + CalculatedFieldTypeTranslations, + getCalculatedFieldArgumentsEditorCompleter, + getCalculatedFieldArgumentsHighlights, + OutputType, + OutputTypeTranslations +} from '@shared/models/calculated-field.models'; +import { digitsRegex, oneSpaceInsideRegex } from '@shared/models/regex.constants'; +import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; +import { EntityType } from '@shared/models/entity-type.models'; +import { map, startWith, switchMap } from 'rxjs/operators'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { ScriptLanguage } from '@shared/models/rule-node.models'; +import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; +import { Observable } from 'rxjs'; + +@Component({ + selector: 'tb-calculated-field-dialog', + templateUrl: './calculated-field-dialog.component.html', + styleUrls: ['./calculated-field-dialog.component.scss'], + encapsulation: ViewEncapsulation.None +}) +export class CalculatedFieldDialogComponent extends DialogComponent { + + fieldFormGroup = this.fb.group({ + name: ['', [Validators.required, Validators.pattern(oneSpaceInsideRegex), Validators.maxLength(255)]], + type: [CalculatedFieldType.SIMPLE], + debugSettings: [], + configuration: this.fb.group({ + arguments: this.fb.control({}), + expressionSIMPLE: ['', [Validators.required, Validators.pattern(oneSpaceInsideRegex), Validators.maxLength(255)]], + expressionSCRIPT: [calculatedFieldDefaultScript], + output: this.fb.group({ + name: ['', [Validators.required, Validators.pattern(oneSpaceInsideRegex), Validators.maxLength(255)]], + scope: [{ value: AttributeScope.SERVER_SCOPE, disabled: true }], + type: [OutputType.Timeseries], + decimalsByDefault: [null as number, [Validators.min(0), Validators.max(15), Validators.pattern(digitsRegex)]], + }), + }), + }); + + functionArgs$ = this.configFormGroup.get('arguments').valueChanges + .pipe( + startWith(this.data.value?.configuration?.arguments ?? {}), + map(argumentsObj => ['ctx', ...Object.keys(argumentsObj)]) + ); + + argumentsEditorCompleter$ = this.configFormGroup.get('arguments').valueChanges + .pipe( + startWith(this.data.value?.configuration?.arguments ?? {}), + map(argumentsObj => getCalculatedFieldArgumentsEditorCompleter(argumentsObj)) + ); + + argumentsHighlightRules$ = this.configFormGroup.get('arguments').valueChanges + .pipe( + startWith(this.data.value?.configuration?.arguments ?? {}), + map(argumentsObj => getCalculatedFieldArgumentsHighlights(argumentsObj)) + ); + + additionalDebugActionConfig = this.data.value?.id ? { + ...this.data.additionalDebugActionConfig, + action: () => this.data.additionalDebugActionConfig.action({ id: this.data.value.id, ...this.fromGroupValue }), + } : null; + + readonly OutputTypeTranslations = OutputTypeTranslations; + readonly OutputType = OutputType; + readonly AttributeScope = AttributeScope; + readonly EntityType = EntityType; + readonly CalculatedFieldType = CalculatedFieldType; + readonly ScriptLanguage = ScriptLanguage; + readonly fieldTypes = Object.values(CalculatedFieldType) as CalculatedFieldType[]; + readonly outputTypes = Object.values(OutputType) as OutputType[]; + readonly CalculatedFieldTypeTranslations = CalculatedFieldTypeTranslations; + + constructor(protected store: Store, + protected router: Router, + @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldDialogData, + protected dialogRef: MatDialogRef, + private calculatedFieldsService: CalculatedFieldsService, + private destroyRef: DestroyRef, + private fb: FormBuilder) { + super(store, router, dialogRef); + this.observeIsLoading(); + this.applyDialogData(); + this.observeTypeChanges(); + } + + get configFormGroup(): FormGroup { + return this.fieldFormGroup.get('configuration') as FormGroup; + } + + get outputFormGroup(): FormGroup { + return this.fieldFormGroup.get('configuration').get('output') as FormGroup; + } + + get fromGroupValue(): CalculatedField { + const { configuration, type, name, ...rest } = this.fieldFormGroup.value; + const { expressionSIMPLE, expressionSCRIPT, output, ...restConfig } = configuration; + return { + configuration: { + ...restConfig, + type, expression: configuration['expression'+type].trim(), + output: { ...output, name: output.name?.trim() ?? '' } + }, + name: name.trim(), + type, + ...rest, + } as CalculatedField; + } + + cancel(): void { + this.dialogRef.close(null); + } + + add(): void { + if (this.fieldFormGroup.valid) { + this.calculatedFieldsService.saveCalculatedField({ entityId: this.data.entityId, ...(this.data.value ?? {}), ...this.fromGroupValue}) + .pipe(takeUntilDestroyed(this.destroyRef)) + .subscribe(calculatedField => this.dialogRef.close(calculatedField)); + } + } + + onTestScript(): void { + const calculatedFieldId = this.data.value?.id?.id; + let testScriptDialogResult$: Observable; + + if (calculatedFieldId) { + testScriptDialogResult$ = this.calculatedFieldsService.getLatestCalculatedFieldDebugEvent(calculatedFieldId) + .pipe( + switchMap(event => { + const args = event?.arguments ? JSON.parse(event.arguments) : null; + return this.data.getTestScriptDialogFn(this.fromGroupValue, args, false); + }), + takeUntilDestroyed(this.destroyRef) + ) + } else { + testScriptDialogResult$ = this.data.getTestScriptDialogFn(this.fromGroupValue, null, false); + } + + testScriptDialogResult$.subscribe(expression => { + this.configFormGroup.get('expressionSCRIPT').setValue(expression); + this.configFormGroup.get('expressionSCRIPT').markAsDirty(); + }); + } + + private applyDialogData(): void { + const { configuration = {}, type = CalculatedFieldType.SIMPLE, debugSettings = { failuresEnabled: true, allEnabled: true }, ...value } = this.data.value ?? {}; + const { expression, ...restConfig } = configuration as CalculatedFieldConfiguration; + const updatedConfig = { ...restConfig , ['expression'+type]: expression }; + this.fieldFormGroup.patchValue({ configuration: updatedConfig, type, debugSettings, ...value }, {emitEvent: false}); + } + + private observeTypeChanges(): void { + this.toggleKeyByCalculatedFieldType(this.fieldFormGroup.get('type').value); + this.toggleScopeByOutputType(this.outputFormGroup.get('type').value); + + this.outputFormGroup.get('type').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(type => this.toggleScopeByOutputType(type)); + this.fieldFormGroup.get('type').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(type => this.toggleKeyByCalculatedFieldType(type)); + } + + private toggleScopeByOutputType(type: OutputType): void { + this.outputFormGroup.get('scope')[type === OutputType.Attribute? 'enable' : 'disable']({emitEvent: false}); + } + + private toggleKeyByCalculatedFieldType(type: CalculatedFieldType): void { + if (type === CalculatedFieldType.SIMPLE) { + this.outputFormGroup.get('name').enable({emitEvent: false}); + this.configFormGroup.get('expressionSIMPLE').enable({emitEvent: false}); + this.configFormGroup.get('expressionSCRIPT').disable({emitEvent: false}); + } else { + this.outputFormGroup.get('name').disable({emitEvent: false}); + this.configFormGroup.get('expressionSIMPLE').disable({emitEvent: false}); + this.configFormGroup.get('expressionSCRIPT').enable({emitEvent: false}); + } + } + + private observeIsLoading(): void { + this.isLoading$.pipe(takeUntilDestroyed()).subscribe(loading => { + if (loading) { + this.fieldFormGroup.disable({emitEvent: false}); + } else { + this.fieldFormGroup.enable({emitEvent: false}); + this.toggleScopeByOutputType(this.outputFormGroup.get('type').value); + this.toggleKeyByCalculatedFieldType(this.fieldFormGroup.get('type').value); + if (this.data.isDirty) { + this.fieldFormGroup.markAsDirty(); + } + } + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html new file mode 100644 index 0000000000..4d7f20d6d5 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -0,0 +1,208 @@ + +
    +
    +
    {{ 'calculated-fields.argument-settings' | translate }}
    +
    +
    +
    {{ 'calculated-fields.argument-name' | translate }}
    + + + @if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('required')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('duplicateName')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('pattern')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('maxlength')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('equalCtx')) { + + warning + + } + +
    + +
    +
    {{ 'entity.entity-type' | translate }}
    + + + @for (type of argumentEntityTypes; track type) { + {{ ArgumentEntityTypeTranslations.get(type) | translate }} + } + + +
    + @if (ArgumentEntityTypeParamsMap.has(entityType)) { +
    +
    {{ ArgumentEntityTypeParamsMap.get(entityType).title | translate }}
    + +
    + } +     + +
    +
    {{ 'calculated-fields.argument-type' | translate }}
    + + + @for (type of argumentTypes; track type) { + {{ ArgumentTypeTranslations.get(type) | translate }} + } + + @if (refEntityKeyFormGroup.get('type').hasError('required') && refEntityKeyFormGroup.get('type').touched) { + + warning + + } + +
    + @if (entityFilter.singleEntity?.id || entityType === ArgumentEntityType.Current || entityType === ArgumentEntityType.Tenant) { + @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Attribute) { +
    +
    {{ 'calculated-fields.timeseries-key' | translate }}
    + @if (refEntityKeyFormGroup.get('type').value === ArgumentType.LatestTelemetry) { + + } @else { + + } + + + +
    + } @else { + @if (enableAttributeScopeSelection) { +
    +
    {{ 'calculated-fields.attribute-scope' | translate }}
    + + + + {{ 'calculated-fields.server-attributes' | translate }} + + + {{ 'calculated-fields.client-attributes' | translate }} + + + {{ 'calculated-fields.shared-attributes' | translate }} + + + +
    + } +
    +
    {{ 'calculated-fields.attribute-key' | translate }}
    + +
    + } + } +     + @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Rolling) { +
    +
    {{ 'calculated-fields.default-value' | translate }}
    + + + +
    + } @else { +
    +
    {{ 'calculated-fields.time-window' | translate }}
    + +
    + @if (maxDataPointsPerRollingArg) { +
    +
    {{ 'calculated-fields.limit' | translate }}
    +
    + + + + + + +
    +
    + } + } +
    +
    +
    + + +
    +
    diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss new file mode 100644 index 0000000000..773489ee60 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@use '../../../../../../../scss/constants' as constants; + +$panel-width: 520px; + +:host { + display: flex; + width: $panel-width; + max-width: 100%; + max-height: 100vh; + + .fixed-title-width { + @media #{constants.$mat-xs} { + min-width: 120px; + } + } + + .limit-field-row { + @media screen and (max-width: $panel-width) { + display: flex; + flex-direction: column; + + .fixed-title-width { + align-self: flex-start; + padding-top: 8px; + } + } + } +} + +:host ::ng-deep { + .time-interval-field { + .advanced-input { + flex-direction: column; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts new file mode 100644 index 0000000000..482851c59c --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -0,0 +1,257 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { ChangeDetectorRef, Component, Input, OnInit, output } from '@angular/core'; +import { TbPopoverComponent } from '@shared/components/popover.component'; +import { FormBuilder, FormControl, FormGroup, ValidatorFn, Validators } from '@angular/forms'; +import { charsWithNumRegex, oneSpaceInsideRegex } from '@shared/models/regex.constants'; +import { + ArgumentEntityType, + ArgumentEntityTypeParamsMap, + ArgumentEntityTypeTranslations, + ArgumentType, + ArgumentTypeTranslations, + CalculatedFieldArgumentValue, + CalculatedFieldType, + getCalculatedFieldCurrentEntityFilter +} from '@shared/models/calculated-field.models'; +import { debounceTime, delay, distinctUntilChanged, filter } from 'rxjs/operators'; +import { EntityType } from '@shared/models/entity-type.models'; +import { AttributeScope, DataKeyType } from '@shared/models/telemetry/telemetry.models'; +import { DatasourceType } from '@shared/models/widget.models'; +import { EntityId } from '@shared/models/id/entity-id'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { EntityFilter } from '@shared/models/query/query.models'; +import { AliasFilterType } from '@shared/models/alias.models'; +import { merge } from 'rxjs'; +import { MINUTE } from '@shared/models/time/time.models'; +import { getCurrentAuthState } from '@core/auth/auth.selectors'; +import { AppState } from '@core/core.state'; +import { Store } from '@ngrx/store'; + +@Component({ + selector: 'tb-calculated-field-argument-panel', + templateUrl: './calculated-field-argument-panel.component.html', + styleUrls: ['./calculated-field-argument-panel.component.scss'] +}) +export class CalculatedFieldArgumentPanelComponent implements OnInit { + + @Input() buttonTitle: string; + @Input() index: number; + @Input() argument: CalculatedFieldArgumentValue; + @Input() entityId: EntityId; + @Input() tenantId: string; + @Input() entityName: string; + @Input() calculatedFieldType: CalculatedFieldType; + @Input() usedArgumentNames: string[]; + + argumentsDataApplied = output<{ value: CalculatedFieldArgumentValue, index: number }>(); + + readonly maxDataPointsPerRollingArg = getCurrentAuthState(this.store).maxDataPointsPerRollingArg; + readonly defaultLimit = Math.floor(this.maxDataPointsPerRollingArg / 10); + + argumentFormGroup = this.fb.group({ + argumentName: ['', [Validators.required, this.uniqNameRequired(), this.notEqualCtxValidator(), Validators.pattern(charsWithNumRegex), Validators.maxLength(255)]], + refEntityId: this.fb.group({ + entityType: [ArgumentEntityType.Current], + id: [''] + }), + refEntityKey: this.fb.group({ + type: [ArgumentType.LatestTelemetry, [Validators.required]], + key: ['', [Validators.pattern(oneSpaceInsideRegex)]], + scope: [{ value: AttributeScope.SERVER_SCOPE, disabled: true }, [Validators.required]], + }), + defaultValue: ['', [Validators.pattern(oneSpaceInsideRegex)]], + limit: [{ value: this.defaultLimit, disabled: !this.maxDataPointsPerRollingArg }], + timeWindow: [MINUTE * 15], + }); + + argumentTypes: ArgumentType[]; + entityFilter: EntityFilter; + + readonly argumentEntityTypes = Object.values(ArgumentEntityType) as ArgumentEntityType[]; + readonly ArgumentEntityTypeTranslations = ArgumentEntityTypeTranslations; + readonly ArgumentType = ArgumentType; + readonly DataKeyType = DataKeyType; + readonly EntityType = EntityType; + readonly datasourceType = DatasourceType; + readonly ArgumentTypeTranslations = ArgumentTypeTranslations; + readonly AttributeScope = AttributeScope; + readonly ArgumentEntityType = ArgumentEntityType; + readonly ArgumentEntityTypeParamsMap = ArgumentEntityTypeParamsMap; + + private currentEntityFilter: EntityFilter; + + constructor( + private fb: FormBuilder, + private cd: ChangeDetectorRef, + private popover: TbPopoverComponent, + private store: Store + ) { + this.observeEntityFilterChanges(); + this.observeEntityTypeChanges() + this.observeEntityKeyChanges(); + this.observeUpdatePosition(); + } + + get entityType(): ArgumentEntityType { + return this.argumentFormGroup.get('refEntityId').get('entityType').value; + } + + get refEntityIdFormGroup(): FormGroup { + return this.argumentFormGroup.get('refEntityId') as FormGroup; + } + + get refEntityKeyFormGroup(): FormGroup { + return this.argumentFormGroup.get('refEntityKey') as FormGroup; + } + + get enableAttributeScopeSelection(): boolean { + return this.entityType === ArgumentEntityType.Device + || (this.entityType === ArgumentEntityType.Current + && (this.entityId.entityType === EntityType.DEVICE || this.entityId.entityType === EntityType.DEVICE_PROFILE)) + } + + ngOnInit(): void { + this.argumentFormGroup.patchValue(this.argument, {emitEvent: false}); + this.currentEntityFilter = getCalculatedFieldCurrentEntityFilter(this.entityName, this.entityId); + this.updateEntityFilter(this.argument.refEntityId?.entityType, true); + this.toggleByEntityKeyType(this.argument.refEntityKey?.type); + this.setInitialEntityKeyType(); + + this.argumentTypes = Object.values(ArgumentType) + .filter(type => type !== ArgumentType.Rolling || this.calculatedFieldType === CalculatedFieldType.SCRIPT); + } + + saveArgument(): void { + const { refEntityId, ...restConfig } = this.argumentFormGroup.value; + const value = (refEntityId.entityType === ArgumentEntityType.Current ? restConfig : { refEntityId, ...restConfig }) as CalculatedFieldArgumentValue; + if (refEntityId.entityType === ArgumentEntityType.Tenant) { + refEntityId.id = this.tenantId; + } + if (value.defaultValue) { + value.defaultValue = value.defaultValue.trim(); + } + value.refEntityKey.key = value.refEntityKey.key.trim(); + this.argumentsDataApplied.emit({ value, index: this.index }); + } + + cancel(): void { + this.popover.hide(); + } + + private toggleByEntityKeyType(type: ArgumentType): void { + const isAttribute = type === ArgumentType.Attribute; + const isRolling = type === ArgumentType.Rolling; + this.argumentFormGroup.get('refEntityKey').get('scope')[isAttribute? 'enable' : 'disable']({ emitEvent: false }); + this.argumentFormGroup.get('limit')[isRolling? 'enable' : 'disable']({ emitEvent: false }); + this.argumentFormGroup.get('timeWindow')[isRolling? 'enable' : 'disable']({ emitEvent: false }); + this.argumentFormGroup.get('defaultValue')[isRolling? 'disable' : 'enable']({ emitEvent: false }); + } + + private updateEntityFilter(entityType: ArgumentEntityType = ArgumentEntityType.Current, onInit = false): void { + let entityFilter: EntityFilter; + switch (entityType) { + case ArgumentEntityType.Current: + entityFilter = this.currentEntityFilter; + break; + case ArgumentEntityType.Tenant: + entityFilter = { + type: AliasFilterType.singleEntity, + singleEntity: { + id: this.tenantId, + entityType: EntityType.TENANT + }, + }; + break; + default: + entityFilter = { + type: AliasFilterType.singleEntity, + singleEntity: this.argumentFormGroup.get('refEntityId').value as unknown as EntityId, + }; + } + if (!onInit) { + this.argumentFormGroup.get('refEntityKey').get('key').setValue(''); + } + this.entityFilter = entityFilter; + this.cd.markForCheck(); + } + + private observeEntityFilterChanges(): void { + merge( + this.refEntityIdFormGroup.get('entityType').valueChanges, + this.refEntityKeyFormGroup.get('type').valueChanges, + this.refEntityIdFormGroup.get('id').valueChanges.pipe(filter(Boolean)), + this.refEntityKeyFormGroup.get('scope').valueChanges, + ) + .pipe(debounceTime(50), takeUntilDestroyed()) + .subscribe(() => this.updateEntityFilter(this.entityType)); + } + + private observeEntityTypeChanges(): void { + this.argumentFormGroup.get('refEntityId').get('entityType').valueChanges + .pipe(distinctUntilChanged(), takeUntilDestroyed()) + .subscribe(type => { + this.argumentFormGroup.get('refEntityId').get('id').setValue(''); + this.argumentFormGroup.get('refEntityId') + .get('id')[type === ArgumentEntityType.Tenant || type === ArgumentEntityType.Current ? 'disable' : 'enable'](); + if (!this.enableAttributeScopeSelection) { + this.refEntityKeyFormGroup.get('scope').setValue(AttributeScope.SERVER_SCOPE); + } + }); + } + + private uniqNameRequired(): ValidatorFn { + return (control: FormControl) => { + const newName = control.value.trim().toLowerCase(); + const isDuplicate = this.usedArgumentNames?.some(name => name.toLowerCase() === newName); + + return isDuplicate ? { duplicateName: true } : null; + }; + } + + private observeEntityKeyChanges(): void { + this.argumentFormGroup.get('refEntityKey').get('type').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(type => this.toggleByEntityKeyType(type)); + } + + private setInitialEntityKeyType(): void { + if (this.calculatedFieldType === CalculatedFieldType.SIMPLE && this.argument.refEntityKey?.type === ArgumentType.Rolling) { + const typeControl = this.argumentFormGroup.get('refEntityKey').get('type'); + typeControl.setValue(null); + typeControl.markAsTouched(); + } + } + + private notEqualCtxValidator(): ValidatorFn { + return (control: FormControl) => { + const trimmedValue = control.value.trim().toLowerCase(); + return trimmedValue === 'ctx' ? { equalCtx: true } : null; + }; + } + + private observeUpdatePosition(): void { + merge( + this.refEntityIdFormGroup.get('entityType').valueChanges, + this.refEntityKeyFormGroup.get('type').valueChanges, + this.argumentFormGroup.get('timeWindow').valueChanges, + this.refEntityIdFormGroup.get('id').valueChanges.pipe(filter(Boolean)), + ) + .pipe(delay(50), takeUntilDestroyed()) + .subscribe(() => this.popover.updatePosition()); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts new file mode 100644 index 0000000000..9e3c52bc4f --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts @@ -0,0 +1,21 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +export * from './dialog/calculated-field-dialog.component'; +export * from './arguments-table/calculated-field-arguments-table.component'; +export * from './panel/calculated-field-argument-panel.component'; +export * from './debug-dialog/calculated-field-debug-dialog.component'; +export * from './test-dialog/calculated-field-script-test-dialog.component'; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html new file mode 100644 index 0000000000..f393a9130b --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html @@ -0,0 +1,58 @@ + +
    +
    {{ 'calculated-fields.arguments' | translate }}
    +
    +
    +
    {{ 'common.name' | translate }}
    +
    {{ 'common.type' | translate }}
    +
    {{ 'common.data' | translate }}
    +
    +
    + @for (group of argumentsFormArray.controls; track group) { +
    + + + + + + + {{ ArgumentTypeTranslations.get(argumentsTypeMap.get(group.get('argumentName').value)) | translate }} + + + +
    + @if (argumentsTypeMap.get(group.get('argumentName').value) === ArgumentType.Rolling) { + + + + } @else { + + + + + } + +
    +
    + } +
    +
    +
    diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.scss new file mode 100644 index 0000000000..cbca3002aa --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.scss @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@use '../../../../../../../scss/constants' as constants; + +:host { + .tb-form-table { + min-width: 700px; + } +} + +:host::ng-deep { + .tb-form-table-row { + .argument-value { + .tb-value-type.row { + width: 120px; + min-width: 120px; + } + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts new file mode 100644 index 0000000000..7c5580f11a --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts @@ -0,0 +1,146 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input } from '@angular/core'; +import { + ControlValueAccessor, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + Validator, + ValidationErrors, + FormBuilder, + FormGroup +} from '@angular/forms'; +import { PageComponent } from '@shared/components/page.component'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { entityTypeTranslations } from '@shared/models/entity-type.models'; +import { + ArgumentType, + ArgumentTypeTranslations, + CalculatedFieldArgumentEventValue, + CalculatedFieldRollingTelemetryArgumentValue, + CalculatedFieldSingleArgumentValue, + CalculatedFieldEventArguments, + CalculatedFieldType +} from '@shared/models/calculated-field.models'; +import { + JsonObjectEditDialogComponent, + JsonObjectEditDialogData +} from '@shared/components/dialog/json-object-edit-dialog.component'; +import { filter } from 'rxjs/operators'; +import { MatDialog } from '@angular/material/dialog'; + +@Component({ + selector: 'tb-calculated-field-test-arguments', + templateUrl: './calculated-field-test-arguments.component.html', + styleUrls: ['./calculated-field-test-arguments.component.scss'], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => CalculatedFieldTestArgumentsComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => CalculatedFieldTestArgumentsComponent), + multi: true, + } + ] +}) +export class CalculatedFieldTestArgumentsComponent extends PageComponent implements ControlValueAccessor, Validator { + + @Input() argumentsTypeMap: Map; + + argumentsFormArray = this.fb.array([]); + + readonly entityTypeTranslations = entityTypeTranslations; + readonly ArgumentTypeTranslations = ArgumentTypeTranslations; + readonly ArgumentType = ArgumentType; + readonly CalculatedFieldType = CalculatedFieldType; + + private propagateChange: (value: CalculatedFieldEventArguments) => void = () => {}; + + constructor(private fb: FormBuilder, private dialog: MatDialog) { + super(); + this.argumentsFormArray.valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(() => this.propagateChange(this.getValue())); + } + + registerOnChange(propagateChange: (value: CalculatedFieldEventArguments) => void): void { + this.propagateChange = propagateChange; + } + + registerOnTouched(_): void { + } + + writeValue(argumentsObj: CalculatedFieldEventArguments): void { + this.argumentsFormArray.clear(); + Object.keys(argumentsObj).forEach(key => { + const value = { ...argumentsObj[key], argumentName: key } as CalculatedFieldArgumentEventValue; + this.argumentsFormArray.push(this.argumentsTypeMap.get(key) === ArgumentType.Rolling + ? this.getRollingArgumentFormGroup(value as CalculatedFieldRollingTelemetryArgumentValue) + : this.getSimpleArgumentFormGroup(value as CalculatedFieldSingleArgumentValue) + ); + }); + } + + validate(): ValidationErrors | null { + return this.argumentsFormArray.valid ? null : { arguments: { valid: false } }; + } + + openEditJSONDialog(group: FormGroup): void { + this.dialog.open(JsonObjectEditDialogComponent, { + disableClose: true, + height: '760px', + maxHeight: '70vh', + minWidth: 'min(700px, 100%)', + panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], + data: { + jsonValue: this.argumentsTypeMap.get(group.get('argumentName').value) === ArgumentType.Rolling ? group.value.rollingJson : group.value, + required: true, + fillHeight: true + } + }).afterClosed() + .pipe(filter(Boolean)) + .subscribe(result => this.argumentsTypeMap.get(group.get('argumentName').value) === ArgumentType.Rolling + ? group.get('rollingJson').patchValue({ values: (result as CalculatedFieldRollingTelemetryArgumentValue).values, timeWindow: (result as CalculatedFieldRollingTelemetryArgumentValue).timeWindow }) + : group.patchValue({ ts: (result as CalculatedFieldSingleArgumentValue).ts, value: (result as CalculatedFieldSingleArgumentValue).value }) ); + } + + private getSimpleArgumentFormGroup({ argumentName, ts, value }: CalculatedFieldSingleArgumentValue): FormGroup { + return this.fb.group({ + argumentName: [{ value: argumentName, disabled: true}], + ts: [ts], + value: [value] + }) as FormGroup; + } + + private getRollingArgumentFormGroup({ argumentName, timeWindow, values }: CalculatedFieldRollingTelemetryArgumentValue): FormGroup { + return this.fb.group({ + argumentName: [{ value: argumentName, disabled: true }], + rollingJson: [{ values: values ?? [], timeWindow: timeWindow ?? {} }] + }) as FormGroup; + } + + private getValue(): CalculatedFieldEventArguments { + return this.argumentsFormArray.getRawValue().reduce((acc, rowItem) => { + const { argumentName, rollingJson = {}, ...value } = rowItem; + acc[argumentName] = { ...rollingJson, ...value }; + return acc; + }, {}); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html new file mode 100644 index 0000000000..6859ff6dab --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html @@ -0,0 +1,103 @@ + +
    + +

    {{ 'calculated-fields.test-script-function' | translate }} ({{ 'api-usage.tbel' | translate }})

    + +     +
    +
    +
    +
    +
    +
    + {{ 'calculated-fields.expression' | translate }} +
    + +
    +
    +
    +
    +
    +
    + {{ 'calculated-fields.arguments' | translate }} +
    + +
    +
    +
    +
    +
    + common.output +
    + +
    +
    +
    +
    +
    +
    +
    + + + + +
    +
    diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss new file mode 100644 index 0000000000..94766e944a --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss @@ -0,0 +1,91 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.cf-test-dialog-container { + .gutter { + background-color: #eee; + background-repeat: no-repeat; + background-position: 50%; + } + + .gutter.gutter-horizontal { + cursor: col-resize; + background-image: url("../../../../../../../assets/split.js/grips/horizontal.png"); + } + + .gutter.gutter-vertical { + cursor: row-resize; + background-image: url("../../../../../../../assets/split.js/grips/vertical.png"); + } + + .block-label { + padding: 4px; + color: #00acc1; + background: rgba(220, 220, 220, .35); + border-radius: 5px; + } + + .test-block-content { + padding-top: 5px; + padding-left: 5px; + border: 1px solid #c0c0c0; + overflow: scroll; + } + + .block-label-container { + position: absolute; + z-index: 10; + font-size: 12px; + font-weight: bold; + + &.left { + right: 112px; + top: 9px; + } + + &.right-bottom { + right: 40px; + top: 6px; + } + + &.right-top { + right: 8px; + top: 2px; + } + } +} + +.tb-js-func { + .ace_tb { + &.ace_calculated-field { + &-ctx { + color: #C52F00; + } + &-args { + color: #185F2A; + } + &-key { + color: #c24c1a; + } + &-time-window, &-values, &-func, &-value, &-ts { + color: #7214D0; + } + &-start-ts, &-end-ts { + color: #2CAA00; + } + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts new file mode 100644 index 0000000000..4e15e4a240 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts @@ -0,0 +1,230 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + AfterViewInit, + Component, + DestroyRef, + ElementRef, + Inject, + OnDestroy, + ViewChild, + ViewEncapsulation, +} from '@angular/core'; +import { MAT_DIALOG_DATA, MatDialog, MatDialogRef } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormBuilder, Validators } from '@angular/forms'; +import { NEVER, Observable, of, switchMap } from 'rxjs'; +import { Router } from '@angular/router'; +import { DialogComponent } from '@shared/components/dialog.component'; +import { ContentType } from '@shared/models/constants'; +import { JsonContentComponent } from '@shared/components/json-content.component'; +import { ScriptLanguage } from '@shared/models/rule-node.models'; +import { ActionNotificationShow } from '@core/notification/notification.actions'; +import { beautifyJs } from '@shared/models/beautify.models'; +import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { filter } from 'rxjs/operators'; +import { + ArgumentType, + CalculatedFieldEventArguments, + CalculatedFieldTestScriptDialogData, + TestArgumentTypeMap +} from '@shared/models/calculated-field.models'; + +@Component({ + selector: 'tb-calculated-field-script-test-dialog', + templateUrl: './calculated-field-script-test-dialog.component.html', + styleUrls: ['./calculated-field-script-test-dialog.component.scss'], + encapsulation: ViewEncapsulation.None +}) +export class CalculatedFieldScriptTestDialogComponent extends DialogComponent implements AfterViewInit, OnDestroy { + + @ViewChild('leftPanel', {static: true}) leftPanelElmRef: ElementRef; + @ViewChild('rightPanel', {static: true}) rightPanelElmRef: ElementRef; + @ViewChild('topRightPanel', {static: true}) topRightPanelElmRef: ElementRef; + @ViewChild('bottomRightPanel', {static: true}) bottomRightPanelElmRef: ElementRef; + @ViewChild('testScriptContainer', {static: true}) testScriptContainer: ElementRef; + + @ViewChild('expressionContent', {static: true}) expressionContent: JsonContentComponent; + + calculatedFieldScriptTestFormGroup = this.fb.group({ + expression: ['', Validators.required], + arguments: [], + output: [] + }); + argumentsTypeMap = new Map(); + + readonly ContentType = ContentType; + readonly ScriptLanguage = ScriptLanguage; + readonly functionArgs = ['ctx', ...Object.keys(this.data.arguments)]; + + private testScriptResize: ResizeObserver; + private splitObjects: SplitObject[] = []; + + constructor(protected store: Store, + protected router: Router, + @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldTestScriptDialogData, + protected dialogRef: MatDialogRef, + private dialog: MatDialog, + private fb: FormBuilder, + private destroyRef: DestroyRef, + private calculatedFieldService: CalculatedFieldsService) { + super(store, router, dialogRef); + beautifyJs(this.data.expression, {indent_size: 4}).pipe(filter(Boolean), takeUntilDestroyed()).subscribe( + (res) => this.calculatedFieldScriptTestFormGroup.get('expression').patchValue(res, {emitEvent: false}) + ); + this.calculatedFieldScriptTestFormGroup.get('arguments').patchValue(this.getArgumentsValue()); + } + + ngAfterViewInit(): void { + this.observeResize(); + } + + ngOnDestroy(): void { + super.ngOnDestroy(); + this.testScriptResize.disconnect(); + } + + cancel(): void { + this.dialogRef.close(null); + } + + onTestScript(): void { + this.testScript() + .pipe( + switchMap(output => beautifyJs(output, {indent_size: 4})), + takeUntilDestroyed(this.destroyRef) + ) + .subscribe(output => this.calculatedFieldScriptTestFormGroup.get('output').setValue(output)); + } + + save(): void { + this.testScript(true).pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => { + this.calculatedFieldScriptTestFormGroup.get('expression').markAsPristine(); + this.dialogRef.close(this.calculatedFieldScriptTestFormGroup.get('expression').value); + }); + } + + private testScript(onSave = false): Observable { + if (this.checkInputParamErrors()) { + return this.calculatedFieldService.testScript({ + expression: this.calculatedFieldScriptTestFormGroup.get('expression').value, + arguments: this.getTestArguments() + }).pipe( + switchMap(result => { + if (result.error) { + this.store.dispatch(new ActionNotificationShow( + { + message: result.error, + type: 'error' + })); + return NEVER; + } else { + if (onSave && this.data.openCalculatedFieldEdit) { + this.dialog.closeAll(); + } + return of(result.output); + } + }), + ); + } else { + return NEVER; + } + } + + private checkInputParamErrors(): boolean { + this.expressionContent.validateOnSubmit(); + return !this.calculatedFieldScriptTestFormGroup.get('expression').invalid; + } + + private observeResize(): void { + this.testScriptResize = new ResizeObserver(() => { + this.updateSizes(); + }); + + this.testScriptResize.observe(this.testScriptContainer.nativeElement); + } + + private updateSizes(): void { + this.initSplitLayout(this.testScriptContainer.nativeElement.clientWidth <= 960); + } + + private getTestArguments(): CalculatedFieldEventArguments { + const argumentsValue = this.calculatedFieldScriptTestFormGroup.get('arguments').value; + return Object.keys(argumentsValue) + .reduce((acc, key) => { + acc[key] = argumentsValue[key]; + acc[key].type = TestArgumentTypeMap.get(this.argumentsTypeMap.get(key)); + return acc; + }, {}); + } + + private getArgumentsValue(): CalculatedFieldEventArguments { + return Object.keys(this.data.arguments) + .reduce((acc, key) => { + const { type, ...argumentObj } = this.data.arguments[key]; + this.argumentsTypeMap.set(key, type); + acc[key] = argumentObj; + return acc; + }, {}); + } + + private initSplitLayout(smallMode = false): void { + const [leftPanel, rightPanel, topRightPanel, bottomRightPanel] = [ + this.leftPanelElmRef.nativeElement, + this.rightPanelElmRef.nativeElement, + this.topRightPanelElmRef.nativeElement, + this.bottomRightPanelElmRef.nativeElement + ] as unknown as string[]; + + this.splitObjects.forEach(obj => obj.destroy()); + this.splitObjects = []; + + if (smallMode) { + this.splitObjects.push( + Split([leftPanel, rightPanel], { + sizes: [33, 67], + gutterSize: 8, + cursor: 'row-resize', + direction: 'vertical' + }), + Split([topRightPanel, bottomRightPanel], { + sizes: [50, 50], + gutterSize: 8, + cursor: 'row-resize', + direction: 'vertical' + }), + ); + } else { + this.splitObjects.push( + Split([leftPanel, rightPanel], { + sizes: [50, 50], + gutterSize: 8, + cursor: 'col-resize' + }), + Split([topRightPanel, bottomRightPanel], { + sizes: [50, 50], + gutterSize: 8, + cursor: 'row-resize', + direction: 'vertical' + }) + ); + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.ts b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.ts index 26f0c1f642..852da5f7bf 100644 --- a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.ts @@ -32,7 +32,7 @@ import { EntityDebugSettingsPanelComponent } from './entity-debug-settings-panel import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { BehaviorSubject, of, shareReplay, timer } from 'rxjs'; import { SECOND, MINUTE } from '@shared/models/time/time.models'; -import { EntityDebugSettings } from '@shared/models/entity.models'; +import { AdditionalDebugActionConfig, EntityDebugSettings } from '@shared/models/entity.models'; import { map, switchMap, takeWhile } from 'rxjs/operators'; import { getCurrentAuthState } from '@core/auth/auth.selectors'; import { AppState } from '@core/core.state'; @@ -61,6 +61,7 @@ export class EntityDebugSettingsButtonComponent implements ControlValueAccessor @Input() debugLimitsConfiguration: string; @Input() entityLabel: string; + @Input() additionalActionConfig: AdditionalDebugActionConfig; debugSettingsFormGroup = this.fb.group({ failuresEnabled: [false], @@ -133,11 +134,11 @@ export class EntityDebugSettingsButtonComponent implements ControlValueAccessor ...debugSettings, maxDebugModeDuration: this.maxDebugModeDuration, debugLimitsConfiguration: this.debugLimitsConfiguration, - entityLabel: this.entityLabel + entityLabel: this.entityLabel, + additionalActionConfig: this.additionalActionConfig, }, {}, {}, {}, true); - debugStrategyPopover.tbComponentRef.instance.popover = debugStrategyPopover; debugStrategyPopover.tbComponentRef.instance.onSettingsApplied.subscribe((settings: EntityDebugSettings) => { this.debugSettingsFormGroup.patchValue(settings); this.cd.markForCheck(); diff --git a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html index f7283f0eac..cf8c0ae14d 100644 --- a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html @@ -48,20 +48,32 @@ -
    - - +
    +
    + @if (additionalActionConfig) { + + } +
    +
    + + +
    diff --git a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.ts b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.ts index 5b9060704f..5fd3613296 100644 --- a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.ts @@ -21,7 +21,7 @@ import { Component, EventEmitter, Input, - OnInit + OnInit, } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { TbPopoverComponent } from '@shared/components/popover.component'; @@ -32,7 +32,7 @@ import { SECOND } from '@shared/models/time/time.models'; import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; import { of, shareReplay, timer } from 'rxjs'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; -import { EntityDebugSettings } from '@shared/models/entity.models'; +import { AdditionalDebugActionConfig, EntityDebugSettings } from '@shared/models/entity.models'; import { distinctUntilChanged, map, startWith, switchMap, takeWhile } from 'rxjs/operators'; @Component({ @@ -48,13 +48,13 @@ import { distinctUntilChanged, map, startWith, switchMap, takeWhile } from 'rxjs }) export class EntityDebugSettingsPanelComponent extends PageComponent implements OnInit { - @Input() popover: TbPopoverComponent; @Input({ transform: booleanAttribute }) failuresEnabled = false; @Input({ transform: booleanAttribute }) allEnabled = false; @Input() entityLabel: string; @Input() allEnabledUntil = 0; @Input() maxDebugModeDuration: number; @Input() debugLimitsConfiguration: string; + @Input() additionalActionConfig: AdditionalDebugActionConfig; onFailuresControl = this.fb.control(false); debugAllControl = this.fb.control(false); @@ -82,7 +82,8 @@ export class EntityDebugSettingsPanelComponent extends PageComponent implements onSettingsApplied = new EventEmitter(); constructor(private fb: FormBuilder, - private cd: ChangeDetectorRef) { + private cd: ChangeDetectorRef, + private popover: TbPopoverComponent) { super(); this.debugAllControl.valueChanges.pipe( @@ -107,7 +108,7 @@ export class EntityDebugSettingsPanelComponent extends PageComponent implements } onCancel(): void { - this.popover?.hide(); + this.popover.hide(); } onApply(): void { diff --git a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.html b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.html index 0523539539..4f53d5f8a3 100644 --- a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.html +++ b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.html @@ -183,6 +183,7 @@ [class.lt-lg:!hidden]="column.mobileHide" *matCellDef="let entity; let row = index" [matTooltip]="cellTooltip(entity, column, row)" + #cellMatTooltip="matTooltip" matTooltipPosition="above" [style]="cellStyle(entity, column, row)"> @@ -209,6 +210,8 @@ [copyText]="column.actionCell.onAction(null, entity)" tooltipText="{{ column.actionCell.nameFunction ? column.actionCell.nameFunction(entity) : column.actionCell.name }}" tooltipPosition="above" + (mouseover)="cellMatTooltip.hide()" + (mouseleave)="cellMatTooltip.show()" [icon]="column.actionCell.icon" [style]="column.actionCell.style"> diff --git a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts index 5d171555a0..4be96f5fdd 100644 --- a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts @@ -26,7 +26,8 @@ import { OnDestroy, OnInit, SimpleChanges, - ViewChild + ViewChild, + ViewContainerRef, } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { Store } from '@ngrx/store'; @@ -141,7 +142,8 @@ export class EntitiesTableComponent extends PageComponent implements IEntitiesTa private router: Router, private elementRef: ElementRef, private fb: FormBuilder, - private zone: NgZone) { + private zone: NgZone, + public viewContainerRef: ViewContainerRef) { super(store); } @@ -687,7 +689,7 @@ export class EntitiesTableComponent extends PageComponent implements IEntitiesTa } cellTooltip(entity: BaseData, column: EntityColumn>, row: number) { - if (column instanceof EntityTableColumn) { + if (column instanceof EntityTableColumn || column instanceof EntityLinkTableColumn) { const col = this.entitiesTableConfig.columns.indexOf(column); const index = row * this.entitiesTableConfig.columns.length + col; let res = this.cellTooltipCache[index]; diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 772ef38ab8..56a17cc7ae 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -18,6 +18,7 @@ import { CellActionDescriptorType, DateEntityTableColumn, EntityActionTableColumn, + EntityLinkTableColumn, EntityTableColumn, EntityTableConfig } from '@home/models/entity/entities-table-config.models'; @@ -29,7 +30,7 @@ import { MatDialog } from '@angular/material/dialog'; import { EntityId } from '@shared/models/id/entity-id'; import { EventService } from '@app/core/http/event.service'; import { EventTableHeaderComponent } from '@home/components/event/event-table-header.component'; -import { EntityTypeResource } from '@shared/models/entity-type.models'; +import { EntityType, EntityTypeResource } from '@shared/models/entity-type.models'; import { fromEvent, Observable } from 'rxjs'; import { PageData } from '@shared/models/page/page-data'; import { Direction } from '@shared/models/page/sort-order'; @@ -39,7 +40,7 @@ import { EventContentDialogComponent, EventContentDialogData } from '@home/components/event/event-content-dialog.component'; -import { isEqual, sortObjectKeys } from '@core/utils'; +import { getEntityDetailsPageURL, isEqual, sortObjectKeys } from '@core/utils'; import { DAY, historyInterval, MINUTE } from '@shared/models/time/time.models'; import { Overlay, OverlayConfig, OverlayRef } from '@angular/cdk/overlay'; import { ChangeDetectorRef, EventEmitter, Injector, StaticProvider, ViewContainerRef } from '@angular/core'; @@ -355,6 +356,89 @@ export class EventTableConfig extends EntityTableConfig { '48px') ); break; + case DebugEventType.DEBUG_CALCULATED_FIELD: + this.columns[0].width = '80px'; + this.columns[1].width = '100px'; + this.columns.push( + new EntityLinkTableColumn('entityId', 'event.entity-id', '100px', + (entity) => `${entity.body.entityId.substring(0, 8)}…`, + (entity) => getEntityDetailsPageURL(entity.body.entityId, entity.body.entityType as EntityType), + false, + () => ({padding: '0 12px 0 0'}), + () => ({padding: '0 12px 0 0'}), + (entity) => entity.body.entityId, + { + name: this.translate.instant('event.copy-entity-id'), + icon: 'content_copy', + style: { + padding: '4px', + 'font-size': '16px', + color: 'rgba(0,0,0,.87)' + }, + isEnabled: () => true, + onAction: ($event, entity) => entity.body.entityId, + type: CellActionDescriptorType.COPY_BUTTON + } + ), + new EntityTableColumn('messageId', 'event.message-id', '100px', + (entity) => entity.body.msgId ? `${entity.body.msgId?.substring(0, 8)}…` : '-', + () => ({padding: '0 12px 0 0'}), + false, + () => ({padding: '0 12px 0 0'}), + (entity) => entity.body.msgId, + false, + { + name: this.translate.instant('event.copy-message-id'), + icon: 'content_copy', + style: { + padding: '4px', + 'font-size': '16px', + color: 'rgba(0,0,0,.87)' + }, + isEnabled: (entity) => !!entity.body.msgId, + onAction: (_, entity) => entity.body.msgId, + type: CellActionDescriptorType.COPY_BUTTON + } + ), + new EntityTableColumn('messageType', 'event.message-type', '100px', + (entity) => entity.body.msgType ?? '-', + () => ({padding: '0 12px 0 0'}), + false, + () => ({padding: '0 12px 0 0'}), + (entity) => entity.body.msgType, + ), + new EntityActionTableColumn('arguments', 'event.arguments', + { + name: this.translate.instant('action.view'), + icon: 'more_horiz', + isEnabled: (entity) => entity.body.arguments !== undefined, + onAction: ($event, entity) => this.showContent($event, entity.body.arguments, + 'event.arguments', ContentType.JSON, true) + }, + '48px' + ), + new EntityActionTableColumn('result', 'event.result', + { + name: this.translate.instant('action.view'), + icon: 'more_horiz', + isEnabled: (entity) => entity.body.result !== undefined, + onAction: ($event, entity) => this.showContent($event, entity.body.result, + 'event.result', ContentType.JSON, true) + }, + '48px' + ), + new EntityActionTableColumn('error', 'event.error', + { + name: this.translate.instant('action.view'), + icon: 'more_horiz', + isEnabled: (entity) => entity.body.error && entity.body.error.length > 0, + onAction: ($event, entity) => this.showContent($event, entity.body.error, + 'event.error') + }, + '48px' + ) + ); + break; } if (updateTableColumns) { this.getTable().columnsUpdated(true); @@ -376,6 +460,14 @@ export class EventTableConfig extends EntityTableConfig { }); } break; + case DebugEventType.DEBUG_CALCULATED_FIELD: + this.cellActionDescriptors.push({ + name: this.translate.instant('common.test-with-this-message', {test: this.translate.instant(this.testButtonLabel)}), + icon: 'bug_report', + isEnabled: () => true, + onAction: (_, entity) => this.debugEventSelected.next(entity.body) + }); + break; } this.getTable()?.cellActionDescriptorsUpdated(); } @@ -446,6 +538,17 @@ export class EventTableConfig extends EntityTableConfig { {key: 'errorStr', title: 'event.error'} ); break; + case DebugEventType.DEBUG_CALCULATED_FIELD: + this.filterColumns.push( + {key: 'entityId', title: 'event.entity-id'}, + {key: 'msgId', title: 'event.message-id'}, + {key: 'msgType', title: 'event.message-type'}, + {key: 'arguments', title: 'event.arguments'}, + {key: 'result', title: 'event.result'}, + {key: 'isError', title: 'event.error'}, + {key: 'errorStr', title: 'event.error'} + ); + break; } } diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 7fdcccad64..e680232129 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -183,6 +183,27 @@ import { } from '@home/components/dashboard-page/layout/select-dashboard-breakpoint.component'; import { EntityChipsComponent } from '@home/components/entity/entity-chips.component'; import { DashboardViewComponent } from '@home/components/dashboard-view/dashboard-view.component'; +import { CalculatedFieldsTableComponent } from '@home/components/calculated-fields/calculated-fields-table.component'; +import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; +import { CalculatedFieldDialogComponent } from '@home/components/calculated-fields/components/dialog/calculated-field-dialog.component'; +import { + EntityDebugSettingsButtonComponent +} from '@home/components/entity/debug/entity-debug-settings-button.component'; +import { + CalculatedFieldArgumentsTableComponent +} from '@home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component'; +import { + CalculatedFieldArgumentPanelComponent +} from '@home/components/calculated-fields/components/panel/calculated-field-argument-panel.component'; +import { + CalculatedFieldDebugDialogComponent +} from '@home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component'; +import { + CalculatedFieldScriptTestDialogComponent +} from '@home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component'; +import { + CalculatedFieldTestArgumentsComponent +} from '@home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component'; @NgModule({ declarations: @@ -326,7 +347,14 @@ import { DashboardViewComponent } from '@home/components/dashboard-view/dashboar RateLimitsDetailsDialogComponent, SendNotificationButtonComponent, EntityChipsComponent, - DashboardViewComponent + DashboardViewComponent, + CalculatedFieldsTableComponent, + CalculatedFieldDialogComponent, + CalculatedFieldArgumentsTableComponent, + CalculatedFieldArgumentPanelComponent, + CalculatedFieldDebugDialogComponent, + CalculatedFieldScriptTestDialogComponent, + CalculatedFieldTestArgumentsComponent, ], imports: [ CommonModule, @@ -338,7 +366,8 @@ import { DashboardViewComponent } from '@home/components/dashboard-view/dashboar SnmpDeviceProfileTransportModule, StatesControllerModule, DeviceCredentialsModule, - DeviceProfileCommonModule + DeviceProfileCommonModule, + EntityDebugSettingsButtonComponent ], exports: [ RouterTabsComponent, @@ -463,11 +492,19 @@ import { DashboardViewComponent } from '@home/components/dashboard-view/dashboar RateLimitsDetailsDialogComponent, SendNotificationButtonComponent, EntityChipsComponent, - DashboardViewComponent + DashboardViewComponent, + CalculatedFieldsTableComponent, + CalculatedFieldDialogComponent, + CalculatedFieldArgumentsTableComponent, + CalculatedFieldArgumentPanelComponent, + CalculatedFieldDebugDialogComponent, + CalculatedFieldScriptTestDialogComponent, + CalculatedFieldTestArgumentsComponent, ], providers: [ WidgetComponentService, CustomDialogService, + DurationLeftPipe, {provide: EMBED_DASHBOARD_DIALOG_TOKEN, useValue: EmbedDashboardDialogComponent}, {provide: COMPLEX_FILTER_PREDICATE_DIALOG_COMPONENT_TOKEN, useValue: ComplexFilterPredicateDialogComponent}, {provide: DASHBOARD_PAGE_COMPONENT_TOKEN, useValue: DashboardPageComponent}, diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html index 9f1839afec..64d5039914 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html @@ -229,6 +229,92 @@ +
    + + {{ 'tenant-profile.calculated-fields' | translate }} tenant-profile.unlimited + +
    + + tenant-profile.max-calculated-fields + + + {{ 'tenant-profile.max-calculated-fields-required' | translate}} + + + {{ 'tenant-profile.max-calculated-fields-range' | translate}} + + + + + tenant-profile.max-data-points-per-rolling-arg + + + {{ 'tenant-profile.max-data-points-per-rolling-arg-required' | translate}} + + + {{ 'tenant-profile.max-data-points-per-rolling-arg-range' | translate}} + + + +
    +
    + + tenant-profile.max-arguments-per-cf + + + {{ 'tenant-profile.max-arguments-per-cf-required' | translate}} + + + {{ 'tenant-profile.max-arguments-per-cf-range' | translate}} + + + +
    +
    + + + + tenant-profile.advanced-settings + + + +
    + + tenant-profile.max-state-size + + + {{ 'tenant-profile.max-state-size-required' | translate}} + + + {{ 'tenant-profile.max-state-size-range' | translate}} + + + + + tenant-profile.max-value-argument-size + + + {{ 'tenant-profile.max-value-argument-size-required' | translate}} + + + {{ 'tenant-profile.max-value-argument-size-range' | translate}} + + + +
    +     +     +
    @@ -638,6 +724,12 @@ [type]="rateLimitsType.EDGE_UPLINK_MESSAGES_PER_EDGE_RATE_LIMIT"> +
    + + +
    +
    diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts index 4cce34c502..b1d6652e4a 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts @@ -118,7 +118,13 @@ export class DefaultTenantProfileConfigurationComponent implements ControlValueA edgeEventRateLimits: [null, []], edgeEventRateLimitsPerEdge: [null, []], edgeUplinkMessagesRateLimits: [null, []], - edgeUplinkMessagesRateLimitsPerEdge: [null, []] + edgeUplinkMessagesRateLimitsPerEdge: [null, []], + maxCalculatedFieldsPerEntity: [null, [Validators.required, Validators.min(0)]], + maxArgumentsPerCF: [null, [Validators.required, Validators.min(0)]], + maxDataPointsPerRollingArg: [null, [Validators.required, Validators.min(0)]], + maxStateSizeInKBytes: [null, [Validators.required, Validators.min(0)]], + calculatedFieldDebugEventsRateLimit: [null, []], + maxSingleValueArgumentSizeInKBytes: [null, [Validators.required, Validators.min(0)]], }); this.defaultTenantProfileConfigurationFormGroup.get('smsEnabled').valueChanges.pipe( diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/rate-limits/rate-limits.models.ts b/ui-ngx/src/app/modules/home/components/profile/tenant/rate-limits/rate-limits.models.ts index ab50c967bf..f09f950ee7 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/rate-limits/rate-limits.models.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/rate-limits/rate-limits.models.ts @@ -45,7 +45,8 @@ export enum RateLimitsType { EDGE_EVENTS_RATE_LIMIT = 'EDGE_EVENTS_RATE_LIMIT', EDGE_EVENTS_PER_EDGE_RATE_LIMIT = 'EDGE_EVENTS_PER_EDGE_RATE_LIMIT', EDGE_UPLINK_MESSAGES_RATE_LIMIT = 'EDGE_UPLINK_MESSAGES_RATE_LIMIT', - EDGE_UPLINK_MESSAGES_PER_EDGE_RATE_LIMIT = 'EDGE_UPLINK_MESSAGES_PER_EDGE_RATE_LIMIT' + EDGE_UPLINK_MESSAGES_PER_EDGE_RATE_LIMIT = 'EDGE_UPLINK_MESSAGES_PER_EDGE_RATE_LIMIT', + CALCULATED_FIELD_DEBUG_EVENT_RATE_LIMIT = 'CALCULATED_FIELD_DEBUG_EVENT_RATE_LIMIT', } export const rateLimitsLabelTranslationMap = new Map( @@ -74,6 +75,7 @@ export const rateLimitsLabelTranslationMap = new Map( [RateLimitsType.EDGE_EVENTS_PER_EDGE_RATE_LIMIT, 'tenant-profile.rate-limits.edge-events-per-edge-rate-limit'], [RateLimitsType.EDGE_UPLINK_MESSAGES_RATE_LIMIT, 'tenant-profile.rate-limits.edge-uplink-messages-rate-limit'], [RateLimitsType.EDGE_UPLINK_MESSAGES_PER_EDGE_RATE_LIMIT, 'tenant-profile.rate-limits.edge-uplink-messages-per-edge-rate-limit'], + [RateLimitsType.CALCULATED_FIELD_DEBUG_EVENT_RATE_LIMIT, 'tenant-profile.rate-limits.calculated-field-debug-event-rate-limit'], ] ); @@ -103,6 +105,7 @@ export const rateLimitsDialogTitleTranslationMap = new Map AdvancedPersistenceSettingComponent), - multi: true - },{ - provide: NG_VALIDATORS, - useExisting: forwardRef(() => AdvancedPersistenceSettingComponent), - multi: true - }] -}) -export class AdvancedPersistenceSettingComponent implements ControlValueAccessor, Validator { - - persistenceForm = this.fb.group({ - timeseries: [null], - latest: [null], - webSockets: [null] - }); - - private propagateChange: (value: any) => void = () => {}; - - constructor(private fb: FormBuilder) { - this.persistenceForm.valueChanges.pipe( - takeUntilDestroyed() - ).subscribe(value => this.propagateChange(value)); - } - - registerOnChange(fn: any) { - this.propagateChange = fn; - } - - registerOnTouched(_fn: any) { - } - - setDisabledState(isDisabled: boolean) { - if (isDisabled) { - this.persistenceForm.disable({emitEvent: false}); - } else { - this.persistenceForm.enable({emitEvent: false}); - } - } - - validate(): ValidationErrors | null { - return this.persistenceForm.valid ? null : { - persistenceForm: false - }; - } - - writeValue(value: AdvancedProcessingStrategy) { - this.persistenceForm.patchValue(value, {emitEvent: false}); - } -} diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-processing-setting-row.component.html similarity index 80% rename from ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.html rename to ui-ngx/src/app/modules/home/components/rule-node/action/advanced-processing-setting-row.component.html index 635f200942..168c0f7340 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-processing-setting-row.component.html @@ -15,17 +15,17 @@ limitations under the License. --> -
    +
    {{ title }}
    rule-node-config.save-time-series.strategy - @for (strategy of persistenceStrategies; track strategy) { - {{ PersistenceTypeTranslationMap.get(strategy) | translate }} + @for (strategy of processingStrategies; track strategy) { + {{ ProcessingTypeTranslationMap.get(strategy) | translate }} } - @if(persistenceSettingRowForm.get('type').value === PersistenceType.DEDUPLICATE) { + @if (processingSettingRowForm.get('type').value === ProcessingType.DEDUPLICATE) { AdvancedPersistenceSettingRowComponent), + useExisting: forwardRef(() => AdvancedProcessingSettingRowComponent), multi: true },{ provide: NG_VALIDATORS, - useExisting: forwardRef(() => AdvancedPersistenceSettingRowComponent), + useExisting: forwardRef(() => AdvancedProcessingSettingRowComponent), multi: true }] }) -export class AdvancedPersistenceSettingRowComponent implements ControlValueAccessor, Validator { +export class AdvancedProcessingSettingRowComponent implements ControlValueAccessor, Validator { @Input() title: string; - persistenceSettingRowForm = this.fb.group({ + processingSettingRowForm = this.fb.group({ type: [defaultAdvancedProcessingConfig.type], deduplicationIntervalSecs: [{value: 60, disabled: true}] }); - PersistenceType = ProcessingType; - persistenceStrategies = [ProcessingType.ON_EVERY_MESSAGE, ProcessingType.DEDUPLICATE, ProcessingType.SKIP]; - PersistenceTypeTranslationMap = ProcessingTypeTranslationMap; + ProcessingType = ProcessingType; + processingStrategies = [ProcessingType.ON_EVERY_MESSAGE, ProcessingType.DEDUPLICATE, ProcessingType.SKIP]; + ProcessingTypeTranslationMap = ProcessingTypeTranslationMap; maxDeduplicateTime = maxDeduplicateTimeSecs; private propagateChange: (value: any) => void = () => {}; constructor(private fb: FormBuilder) { - this.persistenceSettingRowForm.get('type').valueChanges.pipe( + this.processingSettingRowForm.get('type').valueChanges.pipe( takeUntilDestroyed() ).subscribe(() => this.updatedValidation()); - this.persistenceSettingRowForm.valueChanges.pipe( + this.processingSettingRowForm.valueChanges.pipe( takeUntilDestroyed() ).subscribe((value) => this.propagateChange(value)); } @@ -83,32 +83,32 @@ export class AdvancedPersistenceSettingRowComponent implements ControlValueAcces setDisabledState(isDisabled: boolean) { if (isDisabled) { - this.persistenceSettingRowForm.disable({emitEvent: false}); + this.processingSettingRowForm.disable({emitEvent: false}); } else { - this.persistenceSettingRowForm.enable({emitEvent: false}); + this.processingSettingRowForm.enable({emitEvent: false}); this.updatedValidation(); } } validate(): ValidationErrors | null { - return this.persistenceSettingRowForm.valid ? null : { - persistenceSettingRow: false + return this.processingSettingRowForm.valid ? null : { + processingSettingRow: false }; } writeValue(value: AdvancedProcessingConfig) { if (isDefinedAndNotNull(value)) { - this.persistenceSettingRowForm.patchValue(value, {emitEvent: false}); + this.processingSettingRowForm.patchValue(value, {emitEvent: false}); } else { - this.persistenceSettingRowForm.patchValue(defaultAdvancedProcessingConfig); + this.processingSettingRowForm.patchValue(defaultAdvancedProcessingConfig); } } private updatedValidation() { - if (this.persistenceSettingRowForm.get('type').value === ProcessingType.DEDUPLICATE) { - this.persistenceSettingRowForm.get('deduplicationIntervalSecs').enable({emitEvent: false}); + if (this.processingSettingRowForm.get('type').value === ProcessingType.DEDUPLICATE) { + this.processingSettingRowForm.get('deduplicationIntervalSecs').enable({emitEvent: false}); } else { - this.persistenceSettingRowForm.get('deduplicationIntervalSecs').disable({emitEvent: false}) + this.processingSettingRowForm.get('deduplicationIntervalSecs').disable({emitEvent: false}) } } } diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-processing-setting.component.html similarity index 53% rename from ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.html rename to ui-ngx/src/app/modules/home/components/rule-node/action/advanced-processing-setting.component.html index 094f6dbc2a..edce8b12a9 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-processing-setting.component.html @@ -15,22 +15,30 @@ limitations under the License. --> -
    +
    - - + + - + + > +
    diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-processing-setting.component.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-processing-setting.component.ts new file mode 100644 index 0000000000..4946de986a --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-processing-setting.component.ts @@ -0,0 +1,121 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + UntypedFormGroup, + ValidationErrors, + Validator +} from '@angular/forms'; +import { Component, DestroyRef, forwardRef, Input, OnInit } from '@angular/core'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { AdvancedProcessingStrategy } from '@home/components/rule-node/action/timeseries-config.models'; +import { coerceBoolean } from '@shared/decorators/coercion'; +import { AttributeAdvancedProcessingStrategy } from '@home/components/rule-node/action/attributes-config.model'; + +@Component({ + selector: 'tb-advanced-processing-settings', + templateUrl: './advanced-processing-setting.component.html', + providers: [{ + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => AdvancedProcessingSettingComponent), + multi: true + },{ + provide: NG_VALIDATORS, + useExisting: forwardRef(() => AdvancedProcessingSettingComponent), + multi: true + }] +}) +export class AdvancedProcessingSettingComponent implements OnInit, ControlValueAccessor, Validator { + + @Input() + @coerceBoolean() + timeseries = false; + + @Input() + @coerceBoolean() + attributes = false; + + @Input() + @coerceBoolean() + latest = false; + + @Input() + @coerceBoolean() + webSockets = false; + + @Input() + @coerceBoolean() + calculatedFields = false; + + processingForm: UntypedFormGroup; + + private propagateChange: (value: any) => void = () => {}; + + constructor(private fb: FormBuilder, + private destroyRef: DestroyRef) { + } + + ngOnInit() { + this.processingForm = this.fb.group({}); + if (this.timeseries) { + this.processingForm.addControl('timeseries', this.fb.control(null, [])); + } + if (this.attributes) { + this.processingForm.addControl('attributes', this.fb.control(null, [])); + } + if (this.latest) { + this.processingForm.addControl('latest', this.fb.control(null, [])); + } + if (this.webSockets) { + this.processingForm.addControl('webSockets', this.fb.control(null, [])); + } + if (this.calculatedFields) { + this.processingForm.addControl('calculatedFields', this.fb.control(null, [])); + } + this.processingForm.valueChanges.pipe( + takeUntilDestroyed(this.destroyRef) + ).subscribe(value => this.propagateChange(value)); + } + + registerOnChange(fn: any) { + this.propagateChange = fn; + } + + registerOnTouched(_fn: any) { + } + + setDisabledState(isDisabled: boolean) { + if (isDisabled) { + this.processingForm.disable({emitEvent: false}); + } else { + this.processingForm.enable({emitEvent: false}); + } + } + + validate(): ValidationErrors | null { + return this.processingForm.valid ? null : { + processingForm: false + }; + } + + writeValue(value: AdvancedProcessingStrategy | AttributeAdvancedProcessingStrategy) { + this.processingForm.patchValue(value, {emitEvent: false}); + } +} diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/attributes-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/attributes-config.component.html index 03ea4fccdd..0de3f8020f 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/attributes-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/attributes-config.component.html @@ -16,11 +16,54 @@ -->
    +
    +
    +
    + rule-node-config.save-attribute.processing-settings +
    + + {{ 'rule-node-config.basic-mode' | translate}} + {{ 'rule-node-config.advanced-mode' | translate }} + +
    + @if(!attributesConfigForm.get('processingSettings.isAdvanced').value) { + + rule-node-config.save-attribute.strategy + + @for (strategy of processingStrategies; track strategy) { + {{ ProcessingTypeTranslationMap.get(strategy) | translate }} + } + + + + @if(attributesConfigForm.get('processingSettings.type').value === ProcessingType.DEDUPLICATE) { + + + } + } @else { + + } +
    +
    - - +
    + rule-node-config.save-attribute.scope +
    - + {{ 'rule-node-config.attributes-scope' | translate }} @@ -29,7 +72,7 @@ - + {{ 'rule-node-config.attributes-scope-value' | translate }}
    diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts index 47c0501b26..7f27bca241 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts @@ -18,7 +18,7 @@ import { Component } from '@angular/core'; import { FormBuilder, FormGroup, Validators } from '@angular/forms'; import { RuleNodeConfigurationComponent } from '@shared/models/rule-node.models'; import { - defaultAdvancedPersistenceStrategy, + defaultAdvancedProcessingStrategy, maxDeduplicateTimeSecs, ProcessingSettings, ProcessingSettingsForm, @@ -37,9 +37,9 @@ export class TimeseriesConfigComponent extends RuleNodeConfigurationComponent { timeseriesConfigForm: FormGroup; - PersistenceType = ProcessingType; - persistenceStrategies = [ProcessingType.ON_EVERY_MESSAGE, ProcessingType.DEDUPLICATE, ProcessingType.WEBSOCKETS_ONLY]; - PersistenceTypeTranslationMap = ProcessingTypeTranslationMap; + ProcessingType = ProcessingType; + processingStrategies = [ProcessingType.ON_EVERY_MESSAGE, ProcessingType.DEDUPLICATE, ProcessingType.WEBSOCKETS_ONLY]; + ProcessingTypeTranslationMap = ProcessingTypeTranslationMap; maxDeduplicateTime = maxDeduplicateTimeSecs @@ -63,14 +63,14 @@ export class TimeseriesConfigComponent extends RuleNodeConfigurationComponent { type: isAdvanced ? ProcessingType.ON_EVERY_MESSAGE : config.processingSettings.type, isAdvanced: isAdvanced, deduplicationIntervalSecs: config.processingSettings?.deduplicationIntervalSecs ?? 60, - advanced: isAdvanced ? config.processingSettings : defaultAdvancedPersistenceStrategy + advanced: isAdvanced ? config.processingSettings : defaultAdvancedProcessingStrategy } } else { processingSettings = { type: ProcessingType.ON_EVERY_MESSAGE, isAdvanced: false, deduplicationIntervalSecs: 60, - advanced: defaultAdvancedPersistenceStrategy + advanced: defaultAdvancedProcessingStrategy }; } return { diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts index f8987b7f0e..9785125999 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts @@ -55,14 +55,15 @@ export interface BasicProcessingSettings { type: ProcessingType; } -export interface DeduplicateProcessingStrategy extends BasicProcessingSettings{ +export interface DeduplicateProcessingStrategy extends BasicProcessingSettings { deduplicationIntervalSecs: number; } -export interface AdvancedProcessingStrategy extends BasicProcessingSettings{ +export interface AdvancedProcessingStrategy extends BasicProcessingSettings { timeseries: AdvancedProcessingConfig; latest: AdvancedProcessingConfig; webSockets: AdvancedProcessingConfig; + calculatedFields: AdvancedProcessingConfig; } export type AdvancedProcessingConfig = WithOptional; @@ -71,8 +72,9 @@ export const defaultAdvancedProcessingConfig: AdvancedProcessingConfig = { type: ProcessingType.ON_EVERY_MESSAGE } -export const defaultAdvancedPersistenceStrategy: Omit = { +export const defaultAdvancedProcessingStrategy: Omit = { timeseries: defaultAdvancedProcessingConfig, latest: defaultAdvancedProcessingConfig, webSockets: defaultAdvancedProcessingConfig, + calculatedFields: defaultAdvancedProcessingConfig, } diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html index 053d0f5006..d80892664d 100644 --- a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html @@ -84,6 +84,9 @@ {{ 'version-control.export-relations' | translate }} + + {{ 'version-control.export-calculated-fields' | translate }} +
    diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts index e0a88a3e09..d55515a959 100644 --- a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts @@ -25,7 +25,11 @@ import { TranslateService } from '@ngx-translate/core'; import { DialogService } from '@core/services/dialog.service'; import { catchError, map, mergeMap } from 'rxjs/operators'; import { Observable, of } from 'rxjs'; -import { EntityTypeVersionCreateConfig, exportableEntityTypes } from '@shared/models/vc.models'; +import { + EntityTypeVersionCreateConfig, + exportableEntityTypes, + typesWithCalculatedFields +} from '@shared/models/vc.models'; import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; @@ -43,6 +47,8 @@ export class AutoCommitSettingsComponent extends PageComponent implements OnInit isReadOnly: Observable; + readonly typesWithCalculatedFields = typesWithCalculatedFields; + constructor(protected store: Store, private adminService: AdminService, private dialogService: DialogService, @@ -104,7 +110,8 @@ export class AutoCommitSettingsComponent extends PageComponent implements OnInit branch: null, saveAttributes: true, saveRelations: false, - saveCredentials: true + saveCredentials: true, + saveCalculatedFields: true, }; const allowed = this.allowedEntityTypes(); let entityType: EntityType = null; @@ -206,7 +213,8 @@ export class AutoCommitSettingsComponent extends PageComponent implements OnInit branch: [config.branch, []], saveRelations: [config.saveRelations, []], saveAttributes: [config.saveAttributes, []], - saveCredentials: [config.saveCredentials, []] + saveCredentials: [config.saveCredentials, []], + saveCalculatedFields: [config.saveCalculatedFields, []] }) } ); diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html index b9a9a6c01c..8fdf22f6cd 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html @@ -72,6 +72,9 @@ {{ 'version-control.export-relations' | translate }} + + {{ 'version-control.export-calculated-fields' | translate }} + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts index 8b16905587..8618e405b0 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts @@ -33,7 +33,8 @@ import { EntityTypeVersionCreateConfig, exportableEntityTypes, SyncStrategy, - syncStrategyTranslationMap + syncStrategyTranslationMap, + typesWithCalculatedFields } from '@shared/models/vc.models'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; @@ -79,6 +80,8 @@ export class EntityTypesVersionCreateComponent extends PageComponent implements loading = true; + readonly typesWithCalculatedFields = typesWithCalculatedFields; + constructor(protected store: Store, private translate: TranslateService, private fb: UntypedFormBuilder, @@ -150,6 +153,7 @@ export class EntityTypesVersionCreateComponent extends PageComponent implements saveRelations: [config.saveRelations, []], saveAttributes: [config.saveAttributes, []], saveCredentials: [config.saveCredentials, []], + saveCalculatedFields: [config.saveCalculatedFields, []], allEntities: [config.allEntities, []], entityIds: [config.entityIds, [Validators.required]] }) @@ -202,6 +206,7 @@ export class EntityTypesVersionCreateComponent extends PageComponent implements saveAttributes: true, saveRelations: true, saveCredentials: true, + saveCalculatedFields: true, allEntities: true, entityIds: [] }; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html index 4736bb1ee2..b877d19b63 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html @@ -72,6 +72,9 @@ {{ 'version-control.load-relations' | translate }} + + {{ 'version-control.load-calculated-fields' | translate }} + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts index 23f09ae977..f06abaf3a8 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts @@ -31,7 +31,8 @@ import { PageComponent } from '@shared/components/page.component'; import { entityTypesWithoutRelatedData, EntityTypeVersionLoadConfig, - exportableEntityTypes + exportableEntityTypes, + typesWithCalculatedFields } from '@shared/models/vc.models'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; @@ -75,6 +76,8 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On loading = true; + readonly typesWithCalculatedFields = typesWithCalculatedFields; + constructor(protected store: Store, private translate: TranslateService, private popoverService: TbPopoverService, @@ -145,6 +148,7 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On loadRelations: [config.loadRelations, []], loadAttributes: [config.loadAttributes, []], loadCredentials: [config.loadCredentials, []], + loadCalculatedFields: [config.loadCalculatedFields, []], removeOtherEntities: [config.removeOtherEntities, []], findExistingEntityByName: [config.findExistingEntityByName, []] }) @@ -180,6 +184,7 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On loadAttributes: true, loadRelations: true, loadCredentials: true, + loadCalculatedFields: true, removeOtherEntities: false, findExistingEntityByName: true }; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html index 10c2dc5e40..da4c31a1b8 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html @@ -47,6 +47,9 @@ {{ 'version-control.export-relations' | translate }} + + {{ 'version-control.export-calculated-fields' | translate }} + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts index 48b2a2a174..0c4b696ad1 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts @@ -20,6 +20,7 @@ import { UntypedFormBuilder, UntypedFormGroup, Validators } from '@angular/forms import { entityTypesWithoutRelatedData, SingleEntityVersionCreateRequest, + typesWithCalculatedFields, VersionCreateRequestType, VersionCreationResult } from '@shared/models/vc.models'; @@ -71,6 +72,8 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni private versionCreateResultSubscription: Subscription; + readonly typesWithCalculatedFields = typesWithCalculatedFields; + constructor(protected store: Store, private entitiesVersionControlService: EntitiesVersionControlService, private cd: ChangeDetectorRef, @@ -115,7 +118,8 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni ? this.createVersionFormGroup.get('saveRelations').value : false, saveAttributes: !entityTypesWithoutRelatedData.has(this.entityId.entityType) ? this.createVersionFormGroup.get('saveAttributes').value : false, - saveCredentials: this.entityId.entityType === EntityType.DEVICE ? this.createVersionFormGroup.get('saveCredentials').value : false + saveCredentials: this.entityId.entityType === EntityType.DEVICE ? this.createVersionFormGroup.get('saveCredentials').value : false, + saveCalculatedFields: typesWithCalculatedFields.has(this.entityId.entityType) ? this.createVersionFormGroup.get('saveCalculatedFields').value : false, }, type: VersionCreateRequestType.SINGLE_ENTITY }; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html index 4bc61757c4..1183223850 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html @@ -36,6 +36,9 @@ {{ 'version-control.load-relations' | translate }} + + {{ 'version-control.load-calculated-fields' | translate }} + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts index df2e8c19ff..4456eb7ce7 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts @@ -79,7 +79,8 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn this.restoreFormGroup = this.fb.group({ loadAttributes: [true, []], loadRelations: [true, []], - loadCredentials: [true, []] + loadCredentials: [true, []], + loadCalculatedFields: [true, []] }); this.entitiesVersionControlService.getEntityDataInfo(this.externalEntityId, this.versionId).subscribe((data) => { this.entityDataInfo = data; @@ -110,7 +111,8 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn config: { loadRelations: this.entityDataInfo.hasRelations ? this.restoreFormGroup.get('loadRelations').value : false, loadAttributes: this.entityDataInfo.hasAttributes ? this.restoreFormGroup.get('loadAttributes').value : false, - loadCredentials: this.entityDataInfo.hasCredentials ? this.restoreFormGroup.get('loadCredentials').value : false + loadCredentials: this.entityDataInfo.hasCredentials ? this.restoreFormGroup.get('loadCredentials').value : false, + loadCalculatedFields: this.entityDataInfo.hasCalculatedFields ? this.restoreFormGroup.get('loadCalculatedFields').value : false }, type: VersionLoadRequestType.SINGLE_ENTITY }; diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/alarm/alarm-count-basic-config.component.html b/ui-ngx/src/app/modules/home/components/widget/config/basic/alarm/alarm-count-basic-config.component.html index 2e7dc37581..59b137c6c0 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/alarm/alarm-count-basic-config.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/alarm/alarm-count-basic-config.component.html @@ -16,6 +16,15 @@ --> + +
    alarm.filter
    diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/alarm/alarm-count-basic-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/config/basic/alarm/alarm-count-basic-config.component.ts index 20ceb9130a..de2959cb0d 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/alarm/alarm-count-basic-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/alarm/alarm-count-basic-config.component.ts @@ -69,6 +69,7 @@ export class AlarmCountBasicConfigComponent extends BasicWidgetConfigComponent { const settings: CountWidgetSettings = {...countDefaultSettings(true), ...(configData.config.settings || {})}; this.alarmCountWidgetConfigForm = this.fb.group({ alarmFilterConfig: [getAlarmFilterConfig(configData.config.datasources), []], + datasources: [configData.config.datasources, []], settings: [settings, []], @@ -81,6 +82,7 @@ export class AlarmCountBasicConfigComponent extends BasicWidgetConfigComponent { } protected prepareOutputConfig(config: any): WidgetConfigComponentData { + this.widgetConfig.config.datasources = config.datasources; setAlarmFilterConfig(config.alarmFilterConfig, this.widgetConfig.config.datasources); this.widgetConfig.config.settings = {...(this.widgetConfig.config.settings || {}), ...config.settings}; diff --git a/ui-ngx/src/app/modules/home/components/widget/config/datasource.component.html b/ui-ngx/src/app/modules/home/components/widget/config/datasource.component.html index 297f0c301d..0219872626 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/datasource.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/config/datasource.component.html @@ -36,7 +36,7 @@ datasourceFormGroup.get('type').value === datasourceType.entity || datasourceFormGroup.get('type').value === datasourceType.entityCount || datasourceFormGroup.get('type').value === datasourceType.alarmCount ? datasourceFormGroup.get('type').value : ''"> - @@ -98,7 +98,7 @@
    > extends BaseEntity public width: string = '0px', public cellContentFunction: CellContentFunction = (entity, property) => entity[property] ? entity[property] : '', public entityURL: (entity) => string, - public sortable: boolean = true) { + public sortable: boolean = true, + public cellStyleFunction: CellStyleFunction = () => ({}), + public headerCellStyleFunction: HeaderCellStyleFunction = () => ({}), + public cellTooltipFunction: CellTooltipFunction = () => undefined, + public actionCell: CellActionDescriptor = null) { super('link', key, title, width, sortable); } } diff --git a/ui-ngx/src/app/modules/home/models/entity/entity-table-component.models.ts b/ui-ngx/src/app/modules/home/models/entity/entity-table-component.models.ts index 3accf6bfb9..6317f267ca 100644 --- a/ui-ngx/src/app/modules/home/models/entity/entity-table-component.models.ts +++ b/ui-ngx/src/app/modules/home/models/entity/entity-table-component.models.ts @@ -20,7 +20,7 @@ import { SafeHtml } from '@angular/platform-browser'; import { PageLink } from '@shared/models/page/page-link'; import { Timewindow } from '@shared/models/time/time.models'; import { EntitiesDataSource } from '@home/models/datasource/entity-datasource'; -import { ElementRef, EventEmitter } from '@angular/core'; +import { ElementRef, EventEmitter, Renderer2, ViewContainerRef } from '@angular/core'; import { TbAnchorComponent } from '@shared/components/tb-anchor.component'; import { MatPaginator } from '@angular/material/paginator'; import { MatSort } from '@angular/material/sort'; @@ -64,6 +64,7 @@ export interface IEntitiesTableComponent { paginator: MatPaginator; sort: MatSort; route: ActivatedRoute; + viewContainerRef: ViewContainerRef; addEnabled(): boolean; clearSelection(): void; diff --git a/ui-ngx/src/app/modules/home/pages/admin/resource/js-library-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/admin/resource/js-library-table-config.resolver.ts index 314ae7809f..9c5dba6dbb 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/resource/js-library-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/resource/js-library-table-config.resolver.ts @@ -16,6 +16,7 @@ import { Injectable } from '@angular/core'; import { + CellActionDescriptor, checkBoxCell, DateEntityTableColumn, EntityTableColumn, @@ -27,7 +28,9 @@ import { ResourceInfo, ResourceSubType, ResourceSubTypeTranslationMap, - ResourceType + ResourceType, + ResourceInfoWithReferences, + toResourceDeleteResult } from '@shared/models/resource.models'; import { EntityType, entityTypeResources } from '@shared/models/entity-type.models'; import { NULL_UUID } from '@shared/models/id/has-uuid'; @@ -42,8 +45,19 @@ import { PageLink } from '@shared/models/page/page-link'; import { EntityAction } from '@home/models/entity/entity-component.models'; import { JsLibraryTableHeaderComponent } from '@home/pages/admin/resource/js-library-table-header.component'; import { JsResourceComponent } from '@home/pages/admin/resource/js-resource.component'; -import { switchMap } from 'rxjs/operators'; +import { catchError, map, switchMap } from 'rxjs/operators'; import { ResourceTabsComponent } from '@home/pages/admin/resource/resource-tabs.component'; +import { forkJoin, of } from 'rxjs'; +import { parseHttpErrorMessage } from '@core/utils'; +import { ActionNotificationShow } from '@core/notification/notification.actions'; +import { MatDialog } from '@angular/material/dialog'; +import { DialogService } from '@core/services/dialog.service'; +import { + ResourcesInUseDialogComponent, + ResourcesInUseDialogData +} from "@shared/components/resource/resources-in-use-dialog.component"; +import { ResourcesDatasource } from "@home/pages/admin/resource/resources-datasource"; +import { AuthUser } from '@shared/models/user.model'; @Injectable() export class JsLibraryTableConfigResolver { @@ -53,6 +67,8 @@ export class JsLibraryTableConfigResolver { constructor(private store: Store, private resourceService: ResourceService, private translate: TranslateService, + private dialog: MatDialog, + private dialogService: DialogService, private router: Router, private datePipe: DatePipe) { @@ -81,20 +97,16 @@ export class JsLibraryTableConfigResolver { entity => checkBoxCell(entity.tenantId.id === NULL_UUID)), ); - this.config.cellActionDescriptors.push( - { - name: this.translate.instant('javascript.download'), - icon: 'file_download', - isEnabled: () => true, - onAction: ($event, entity) => this.downloadResource($event, entity) - } - ); + this.config.cellActionDescriptors = this.configureCellActions(getCurrentAuthUser(this.store)); - this.config.deleteEntityTitle = resource => this.translate.instant('javascript.delete-javascript-resource-title', - { resourceTitle: resource.title }); - this.config.deleteEntityContent = () => this.translate.instant('javascript.delete-javascript-resource-text'); - this.config.deleteEntitiesTitle = count => this.translate.instant('javascript.delete-javascript-resources-title', {count}); - this.config.deleteEntitiesContent = () => this.translate.instant('javascript.delete-javascript-resources-text'); + this.config.groupActionDescriptors = [{ + name: this.translate.instant('action.delete'), + icon: 'delete', + isEnabled: true, + onAction: ($event, entities) => this.deleteResources($event, entities) + }]; + + this.config.entitiesDeleteEnabled = false; this.config.entitiesFetchFunction = pageLink => this.resourceService.getResources(pageLink, ResourceType.JS_MODULE, this.config.componentsData.resourceSubType); this.config.loadEntity = id => { @@ -115,7 +127,6 @@ export class JsLibraryTableConfigResolver { } return saveObservable; }; - this.config.deleteEntity = id => this.resourceService.deleteResource(id.id); this.config.onEntityAction = action => this.onResourceAction(action); } @@ -126,7 +137,6 @@ export class JsLibraryTableConfigResolver { resourceSubType: '' }; const authUser = getCurrentAuthUser(this.store); - this.config.deleteEnabled = (resource) => this.isResourceEditable(resource, authUser.authority); this.config.entitySelectionEnabled = (resource) => this.isResourceEditable(resource, authUser.authority); this.config.detailsReadonly = (resource) => this.detailsReadonly(resource, authUser.authority); return this.config; @@ -170,4 +180,151 @@ export class JsLibraryTableConfigResolver { return authority === Authority.SYS_ADMIN; } } + + private deleteResource($event: Event, resource: ResourceInfo) { + if ($event) { + $event.stopPropagation(); + } + this.dialogService.confirm( + this.translate.instant('javascript.delete-javascript-resource-title', { resourceTitle: resource.title }), + this.translate.instant('javascript.delete-javascript-resource-text'), + this.translate.instant('action.no'), + this.translate.instant('action.yes'), + true + ).subscribe((result) => { + if (result) { + this.resourceService.deleteResource(resource.id.id, false, {ignoreErrors: true}).pipe( + map(() => toResourceDeleteResult(resource)), + catchError((err) => of(toResourceDeleteResult(resource, err))) + ).subscribe( + (deleteResult) => { + if (deleteResult.success) { + this.config.updateData(); + } else if (deleteResult.resourceIsReferencedError) { + const resources: ResourceInfoWithReferences[] = [{...resource, ...{references: deleteResult.references}}]; + const data = { + multiple: false, + resources, + configuration: { + title: 'javascript.javascript-resource-is-in-use', + message: this.translate.instant('javascript.javascript-resource-is-in-use-text', {title: resources[0].title}), + deleteText: 'javascript.delete-javascript-resource-in-use-text', + selectedText: 'javascript.selected-javascript-resources', + columns: ['select', 'title', 'references'] + } + }; + this.dialog.open(ResourcesInUseDialogComponent, { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], + data + }).afterClosed().subscribe((resources) => { + if (resources) { + this.resourceService.deleteResource(resource.id.id, true).subscribe( + () => { + this.config.updateData(); + } + ); + } + }); + } else { + const errorMessageWithTimeout = parseHttpErrorMessage(deleteResult.error, this.translate); + setTimeout(() => { + this.store.dispatch(new ActionNotificationShow({message: errorMessageWithTimeout.message, type: 'error'})); + }, errorMessageWithTimeout.timeout); + } + } + ); + } + }); + } + + private deleteResources($event: Event, resources: ResourceInfo[]) { + if ($event) { + $event.stopPropagation(); + } + if (resources && resources.length) { + const title = this.translate.instant('javascript.delete-javascript-resources-title', {count: resources.length}); + const content = this.translate.instant('javascript.delete-javascript-resources-text'); + this.dialogService.confirm(title, content, + this.translate.instant('action.no'), + this.translate.instant('action.yes')).subscribe((result) => { + if (result) { + const tasks = resources.map((resource) => + this.resourceService.deleteResource(resource.id.id, false, {ignoreErrors: true}).pipe( + map(() => toResourceDeleteResult(resource)), + catchError((err) => of(toResourceDeleteResult(resource, err))) + ) + ); + forkJoin(tasks).subscribe( + (deleteResults) => { + const anySuccess = deleteResults.some(res => res.success); + const referenceErrors = deleteResults.filter(res => res.resourceIsReferencedError); + const otherError = deleteResults.find(res => !res.success); + if (anySuccess) { + this.config.updateData(); + } + if (referenceErrors?.length) { + const resourcesWithReferences: ResourceInfoWithReferences[] = + referenceErrors.map(ref => ({...ref.resource, ...{references: ref.references}})); + const data = { + multiple: true, + resources: resourcesWithReferences, + configuration: { + title: 'javascript.javascript-resources-are-in-use', + message: this.translate.instant('javascript.javascript-resources-are-in-use-text'), + deleteText: 'javascript.delete-javascript-resource-in-use-text', + selectedText: 'javascript.selected-javascript-resources', + datasource: new ResourcesDatasource(this.resourceService, resourcesWithReferences, entity => true), + columns: ['select', 'title', 'references'] + } + }; + this.dialog.open(ResourcesInUseDialogComponent, { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], + data + }).afterClosed().subscribe((forceDeleteResources) => { + if (forceDeleteResources && forceDeleteResources.length) { + const forceDeleteTasks = forceDeleteResources.map((resource) => + this.resourceService.deleteResource(resource.id.id, true) + ); + forkJoin(forceDeleteTasks).subscribe( + () => { + this.config.updateData(); + } + ); + } + }); + } else if (otherError) { + const errorMessageWithTimeout = parseHttpErrorMessage(otherError.error, this.translate); + setTimeout(() => { + this.store.dispatch(new ActionNotificationShow({message: errorMessageWithTimeout.message, type: 'error'})); + }, errorMessageWithTimeout.timeout); + } + } + ); + } + }); + } + } + + private configureCellActions(authUser: AuthUser): Array> { + const actions: Array> = []; + actions.push( + { + name: this.translate.instant('javascript.download'), + icon: 'file_download', + isEnabled: () => true, + onAction: ($event, entity) => this.downloadResource($event, entity) + }, + { + name: this.translate.instant('javascript.delete'), + icon: 'delete', + isEnabled: (resource) => this.isResourceEditable(resource, authUser.authority), + onAction: ($event, entity) => this.deleteResource($event, entity) + }, + ); + return actions; + } } diff --git a/ui-ngx/src/app/modules/home/pages/admin/resource/resources-datasource.ts b/ui-ngx/src/app/modules/home/pages/admin/resource/resources-datasource.ts new file mode 100644 index 0000000000..ed9c9f5dd9 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/resource/resources-datasource.ts @@ -0,0 +1,130 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { CollectionViewer, DataSource, SelectionModel } from '@angular/cdk/collections'; +import { ResourceInfo, ResourceSubType, ResourceType } from '@shared/models/resource.models'; +import { BehaviorSubject, Observable, of, ReplaySubject, Subject } from 'rxjs'; +import { emptyPageData, PageData } from '@shared/models/page/page-data'; +import { EntityBooleanFunction } from '@home/models/entity/entities-table-config.models'; +import { PageLink } from '@shared/models/page/page-link'; +import { catchError, map, take, tap } from 'rxjs/operators'; +import { ResourceService } from "@core/http/resource.service"; + +export class ResourcesDatasource implements DataSource { + private entitiesSubject: Subject; + private readonly pageDataSubject: Subject>; + + public pageData$: Observable>; + + public selection = new SelectionModel(true, []); + + public dataLoading = true; + + constructor(private resourceService: ResourceService, + private resources: ResourceInfo[], + private selectionEnabledFunction: EntityBooleanFunction) { + if (this.resources && this.resources.length) { + this.entitiesSubject = new BehaviorSubject(this.resources); + } else { + this.entitiesSubject = new BehaviorSubject([]); + this.pageDataSubject = new BehaviorSubject>(emptyPageData()); + this.pageData$ = this.pageDataSubject.asObservable(); + } + } + + connect(collectionViewer: CollectionViewer): + Observable> { + return this.entitiesSubject.asObservable(); + } + + disconnect(collectionViewer: CollectionViewer): void { + this.entitiesSubject.complete(); + if (this.pageDataSubject) { + this.pageDataSubject.complete(); + } + } + + reset() { + this.entitiesSubject.next([]); + if (this.pageDataSubject) { + this.pageDataSubject.next(emptyPageData()); + } + } + + loadEntities(pageLink: PageLink, resourceType: ResourceType, subType: ResourceSubType): Observable> { + this.dataLoading = true; + const result = new ReplaySubject>(); + this.fetchEntities(pageLink, resourceType, subType).pipe( + tap(() => { + this.selection.clear(); + }), + catchError(() => of(emptyPageData())), + ).subscribe( + (pageData) => { + this.entitiesSubject.next(pageData.data); + this.pageDataSubject.next(pageData); + result.next(pageData); + this.dataLoading = false; + } + ); + return result; + } + + fetchEntities(pageLink: PageLink, resourceType: ResourceType, subType: ResourceSubType): Observable> { + return this.resourceService.getResources(pageLink, resourceType, subType); + } + + isAllSelected(): Observable { + const numSelected = this.selection.selected.length; + return this.entitiesSubject.pipe( + map((entities) => numSelected === entities.length) + ); + } + + isEmpty(): Observable { + return this.entitiesSubject.pipe( + map((entities) => !entities.length) + ); + } + + total(): Observable { + return this.pageDataSubject.pipe( + map((pageData) => pageData.totalElements) + ); + } + + masterToggle() { + this.entitiesSubject.pipe( + tap((entities) => { + const numSelected = this.selection.selected.length; + if (numSelected === this.selectableEntitiesCount(entities)) { + this.selection.clear(); + } else { + entities.forEach(row => { + if (this.selectionEnabledFunction(row)) { + this.selection.select(row); + } + }); + } + }), + take(1) + ).subscribe(); + } + + private selectableEntitiesCount(entities: Array): number { + return entities.filter((entity) => this.selectionEnabledFunction(entity)).length; + } +} diff --git a/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html b/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html index ea93ea4c6b..e4431abfed 100644 --- a/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html @@ -15,6 +15,10 @@ limitations under the License. --> + + + diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html index 0cd3e759b9..767d11eb23 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html @@ -32,6 +32,10 @@ [entityName]="entity.name"> + + + diff --git a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html index c45491a608..46aa08ff9f 100644 --- a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html @@ -40,6 +40,10 @@ + + + + + + diff --git a/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol-editor.models.ts b/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol-editor.models.ts index f6d37d3390..13b55d4f67 100644 --- a/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol-editor.models.ts +++ b/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol-editor.models.ts @@ -31,7 +31,12 @@ import { } from '@home/components/widget/lib/scada/scada-symbol.models'; import { TbEditorCompletion, TbEditorCompletions } from '@shared/models/ace/completion.models'; import { CustomTranslatePipe } from '@shared/pipe/custom-translate.pipe'; -import { AceHighlightRule, AceHighlightRules } from '@shared/models/ace/ace.models'; +import { + AceHighlightRule, + AceHighlightRules, + dotOperatorHighlightRule, + endGroupHighlightRule +} from '@shared/models/ace/ace.models'; import { HelpLinks, ValueType } from '@shared/models/constants'; import { formPropertyCompletions } from '@shared/models/dynamic-form.models'; import ITooltipsterInstance = JQueryTooltipster.ITooltipsterInstance; @@ -921,17 +926,6 @@ export class ScadaSymbolElement { const identifierRe = /[a-zA-Z$_\u00a1-\uffff][a-zA-Z\d$_\u00a1-\uffff]*/; -const dotOperatorHighlightRule: AceHighlightRule = { - token: 'punctuation.operator', - regex: /[.](?![.])/, -}; - -const endGroupHighlightRule: AceHighlightRule = { - regex: '', - token: 'empty', - next: 'no_regex' -}; - const scadaSymbolCtxObjectHighlightRule: AceHighlightRule = { token: 'tb.scada-symbol-ctx', regex: /\bctx\b/, diff --git a/ui-ngx/src/app/shared/components/dialog/json-object-edit-dialog.component.html b/ui-ngx/src/app/shared/components/dialog/json-object-edit-dialog.component.html index 15a462e7f6..5047302bc7 100644 --- a/ui-ngx/src/app/shared/components/dialog/json-object-edit-dialog.component.html +++ b/ui-ngx/src/app/shared/components/dialog/json-object-edit-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
    +

    {{ title }}

    @@ -25,30 +25,26 @@ close     - - -
    -
    -
    - - -
    +
    +
    + +
    diff --git a/ui-ngx/src/app/shared/components/dialog/json-object-edit-dialog.component.ts b/ui-ngx/src/app/shared/components/dialog/json-object-edit-dialog.component.ts index d409e4a80c..d6f733ade7 100644 --- a/ui-ngx/src/app/shared/components/dialog/json-object-edit-dialog.component.ts +++ b/ui-ngx/src/app/shared/components/dialog/json-object-edit-dialog.component.ts @@ -30,6 +30,7 @@ export interface JsonObjectEditDialogData { title?: string; saveLabel?: string; cancelLabel?: string; + fillHeight?: boolean; } @Component({ diff --git a/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html b/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html index ca1fbe412a..af63bae6a7 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html +++ b/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html @@ -15,12 +15,17 @@ limitations under the License. --> - - {{ label | translate }} + {{ label | translate }} {{ displayEntityFn(selectEntityFormGroup.get('entity').value) }} + + warning + + } @else if (keyControl.hasError('required') && keyControl.touched) { + + warning + + } + + @for (key of filteredKeys$ | async; track key) { + + } @empty { + @if (!this.keyControl.value) { + {{ 'entity.no-keys-found' | translate }} + } + } + + diff --git a/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts b/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts new file mode 100644 index 0000000000..84d723d114 --- /dev/null +++ b/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts @@ -0,0 +1,145 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, effect, ElementRef, forwardRef, input, OnChanges, SimpleChanges, ViewChild, } from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; +import { map, startWith, switchMap } from 'rxjs/operators'; +import { combineLatest, of, Subject } from 'rxjs'; +import { EntityService } from '@core/http/entity.service'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { AttributeScope, DataKeyType } from '@shared/models/telemetry/telemetry.models'; +import { EntitiesKeysByQuery } from '@shared/models/entity.models'; +import { EntityFilter } from '@shared/models/query/query.models'; +import { isEqual } from '@core/utils'; + +@Component({ + selector: 'tb-entity-key-autocomplete', + templateUrl: './entity-key-autocomplete.component.html', + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => EntityKeyAutocompleteComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => EntityKeyAutocompleteComponent), + multi: true + } + ], +}) +export class EntityKeyAutocompleteComponent implements ControlValueAccessor, Validator, OnChanges { + + @ViewChild('keyInput', {static: true}) keyInput: ElementRef; + + entityFilter = input.required(); + dataKeyType = input.required(); + keyScopeType = input(); + + keyControl = this.fb.control('', [Validators.required]); + searchText = ''; + keyInputSubject = new Subject(); + + private propagateChange: (value: string) => void; + private cachedResult: EntitiesKeysByQuery; + + keys$ = this.keyInputSubject.asObservable() + .pipe( + switchMap(() => { + return this.cachedResult ? of(this.cachedResult) : this.entityService.findEntityKeysByQuery({ + pageLink: { page: 0, pageSize: 100 }, + entityFilter: this.entityFilter(), + }, this.dataKeyType() === DataKeyType.attribute, this.dataKeyType() === DataKeyType.timeseries, this.keyScopeType()); + }), + map(result => { + this.cachedResult = result; + switch (this.dataKeyType()) { + case DataKeyType.attribute: + return result.attribute; + case DataKeyType.timeseries: + return result.timeseries; + default: + return []; + } + }), + ); + + filteredKeys$ = combineLatest([this.keys$, this.keyControl.valueChanges.pipe(startWith(''))]) + .pipe( + map(([keys, searchText = '']) => { + this.searchText = searchText; + return searchText ? keys.filter(item => item.toLowerCase().includes(searchText.toLowerCase())) : keys; + }) + ); + + constructor( + private fb: FormBuilder, + private entityService: EntityService, + ) { + this.keyControl.valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(value => this.propagateChange(value)); + effect(() => { + if (this.keyScopeType() || this.entityFilter() && this.dataKeyType()) { + this.cachedResult = null; + this.searchText = ''; + } + }); + } + + ngOnChanges(changes: SimpleChanges): void { + const filterChanged = changes.entityFilter?.previousValue && + !isEqual(changes.entityFilter.currentValue, changes.entityFilter.previousValue); + const keyScopeChanged = changes.keyScopeType?.previousValue && + changes.keyScopeType.currentValue !== changes.keyScopeType.previousValue; + const keyTypeChanged = changes.dataKeyType?.previousValue && + changes.dataKeyType.currentValue !== changes.dataKeyType.previousValue; + + if (filterChanged || keyScopeChanged || keyTypeChanged) { + this.keyControl.setValue('', {emitEvent: false}); + } + } + + clear(): void { + this.keyControl.patchValue('', {emitEvent: true}); + setTimeout(() => { + this.keyInput.nativeElement.blur(); + this.keyInput.nativeElement.focus(); + }, 0); + } + + registerOnChange(onChange: (value: string) => void): void { + this.propagateChange = onChange; + } + + registerOnTouched(_): void {} + + validate(): ValidationErrors | null { + return this.keyControl.valid ? null : { keyControl: false }; + } + + writeValue(value: string): void { + this.keyControl.patchValue(value, {emitEvent: false}); + } +} diff --git a/ui-ngx/src/app/shared/components/image/image-gallery.component.ts b/ui-ngx/src/app/shared/components/image/image-gallery.component.ts index ef22c037f2..6e40683c2b 100644 --- a/ui-ngx/src/app/shared/components/image/image-gallery.component.ts +++ b/ui-ngx/src/app/shared/components/image/image-gallery.component.ts @@ -16,10 +16,10 @@ import { ImageResourceInfo, - ImageResourceInfoWithReferences, + ResourceInfoWithReferences, imageResourceType, ResourceSubType, - toImageDeleteResult + toResourceDeleteResult } from '@shared/models/resource.models'; import { forkJoin, merge, Observable, of, Subject, Subscription } from 'rxjs'; import { ImageService } from '@core/http/image.service'; @@ -67,9 +67,9 @@ import { ImageDialogComponent, ImageDialogData } from '@shared/components/image/ import { ImportExportService } from '@shared/import-export/import-export.service'; import { ActionNotificationShow } from '@core/notification/notification.actions'; import { - ImagesInUseDialogComponent, - ImagesInUseDialogData -} from '@shared/components/image/images-in-use-dialog.component'; + ResourcesInUseDialogComponent, + ResourcesInUseDialogData +} from '@shared/components/resource/resources-in-use-dialog.component'; import { ImagesDatasource } from '@shared/components/image/images-datasource'; import { EmbedImageDialogComponent, EmbedImageDialogData } from '@shared/components/image/embed-image-dialog.component'; @@ -504,21 +504,30 @@ export class ImageGalleryComponent extends PageComponent implements OnInit, OnDe this.translate.instant('action.yes')).subscribe((result) => { if (result) { this.imageService.deleteImage(imageResourceType(image), image.resourceKey, false, {ignoreErrors: true}).pipe( - map(() => toImageDeleteResult(image)), - catchError((err) => of(toImageDeleteResult(image, err))) + map(() => toResourceDeleteResult(image)), + catchError((err) => of(toResourceDeleteResult(image, err))) ).subscribe( (deleteResult) => { if (deleteResult.success) { this.imageDeleted(itemIndex); - } else if (deleteResult.imageIsReferencedError) { - this.dialog.open(ImagesInUseDialogComponent, { + } else if (deleteResult.resourceIsReferencedError) { + const images = [{...image, ...{references: deleteResult.references}}]; + const data = { + multiple: false, + resources: images, + configuration: { + title: 'image.image-is-in-use', + message: this.translate.instant('image.image-is-in-use-text', {title: images[0].title}), + deleteText: 'image.delete-image-in-use-text', + selectedText: 'image.selected-images', + columns: ['select', 'preview', 'title', 'references'] + } + }; + this.dialog.open(ResourcesInUseDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], - data: { - multiple: false, - images: [{...image, ...{references: deleteResult.references}}] - } + data }).afterClosed().subscribe((images) => { if (images) { this.imageService.deleteImage(imageResourceType(image), image.resourceKey, true).subscribe( @@ -554,29 +563,38 @@ export class ImageGalleryComponent extends PageComponent implements OnInit, OnDe if (result) { const tasks = selectedImages.map((image) => this.imageService.deleteImage(imageResourceType(image), image.resourceKey, false, {ignoreErrors: true}).pipe( - map(() => toImageDeleteResult(image)), - catchError((err) => of(toImageDeleteResult(image, err))) + map(() => toResourceDeleteResult(image)), + catchError((err) => of(toResourceDeleteResult(image, err))) ) ); forkJoin(tasks).subscribe( (deleteResults) => { const anySuccess = deleteResults.some(res => res.success); - const referenceErrors = deleteResults.filter(res => res.imageIsReferencedError); + const referenceErrors = deleteResults.filter(res => res.resourceIsReferencedError); const otherError = deleteResults.find(res => !res.success); if (anySuccess) { this.updateData(); } if (referenceErrors?.length) { - const imagesWithReferences: ImageResourceInfoWithReferences[] = - referenceErrors.map(ref => ({...ref.image, ...{references: ref.references}})); - this.dialog.open(ImagesInUseDialogComponent, { + const imagesWithReferences: ResourceInfoWithReferences[] = + referenceErrors.map(ref => ({...ref.resource, ...{references: ref.references}})); + const data = { + multiple: true, + resources: imagesWithReferences, + configuration: { + title: 'image.images-are-in-use', + message: this.translate.instant('image.images-are-in-use-text'), + deleteText: 'image.delete-image-in-use-text', + selectedText: 'image.selected-images', + columns: ['select', 'preview', 'title', 'references'], + datasource: new ImagesDatasource(null, imagesWithReferences, entity => true) + } + }; + this.dialog.open(ResourcesInUseDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], - data: { - multiple: true, - images: imagesWithReferences - } + data }).afterClosed().subscribe((forceDeleteImages) => { if (forceDeleteImages && forceDeleteImages.length) { const forceDeleteTasks = forceDeleteImages.map((image) => diff --git a/ui-ngx/src/app/shared/components/image/image-references.component.ts b/ui-ngx/src/app/shared/components/image/image-references.component.ts index af2b4e9798..802408ee3a 100644 --- a/ui-ngx/src/app/shared/components/image/image-references.component.ts +++ b/ui-ngx/src/app/shared/components/image/image-references.component.ts @@ -17,7 +17,7 @@ import { ChangeDetectorRef, Component, Input, OnInit } from '@angular/core'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { ImageReferences } from '@shared/models/resource.models'; +import { ResourceReferences } from '@shared/models/resource.models'; import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; import { TranslateService } from '@ngx-translate/core'; import { getEntityDetailsPageURL } from '@core/utils'; @@ -54,7 +54,7 @@ type ReferencedEntitiesEntry = [string, TenantReferencedEntities]; export class ImageReferencesComponent implements OnInit { @Input() - references: ImageReferences; + references: ResourceReferences; popoverComponent: TbPopoverComponent; @@ -99,7 +99,7 @@ export class ImageReferencesComponent implements OnInit { return tenantId === NULL_UUID; } - private hasNonSystemEntities(references: ImageReferences): boolean { + private hasNonSystemEntities(references: ResourceReferences): boolean { for (const entityTypeStr of Object.keys(references)) { const entities = this.references[entityTypeStr]; if (entities.some(e => e.tenantId && e.tenantId.id && e.tenantId.id !== NULL_UUID)) { @@ -109,7 +109,7 @@ export class ImageReferencesComponent implements OnInit { return false; } - private toReferencedEntitiesList(references: ImageReferences): ReferencedEntityInfo[] { + private toReferencedEntitiesList(references: ResourceReferences): ReferencedEntityInfo[] { const result: ReferencedEntityInfo[] = []; for (const entityTypeStr of Object.keys(references)) { const entityType = entityTypeStr as EntityType; @@ -127,7 +127,7 @@ export class ImageReferencesComponent implements OnInit { return result; } - private toReferencedEntitiesEntries(references: ImageReferences): Observable { + private toReferencedEntitiesEntries(references: ResourceReferences): Observable { let referencedEntities: ReferencedEntities = {}; const referencedEntitiesList = this.toReferencedEntitiesList(references); for (const referencedEntityInfo of referencedEntitiesList) { diff --git a/ui-ngx/src/app/shared/components/js-func.component.ts b/ui-ngx/src/app/shared/components/js-func.component.ts index 504823d40a..eccb5cb908 100644 --- a/ui-ngx/src/app/shared/components/js-func.component.ts +++ b/ui-ngx/src/app/shared/components/js-func.component.ts @@ -20,9 +20,11 @@ import { ElementRef, forwardRef, Input, + OnChanges, OnDestroy, OnInit, Renderer2, + SimpleChanges, ViewChild, ViewContainerRef, ViewEncapsulation @@ -35,7 +37,7 @@ import { ActionNotificationHide, ActionNotificationShow } from '@core/notificati import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { UtilsService } from '@core/services/utils.service'; -import { deepClone, guid, isUndefined, isUndefinedOrNull } from '@app/core/utils'; +import { deepClone, guid, isEqual, isObject, isUndefined, isUndefinedOrNull } from '@app/core/utils'; import { TranslateService } from '@ngx-translate/core'; import { CancelAnimationFrame, RafService } from '@core/services/raf.service'; import { TbEditorCompleter } from '@shared/models/ace/completion.models'; @@ -67,7 +69,7 @@ import { catchError } from 'rxjs/operators'; ], encapsulation: ViewEncapsulation.None }) -export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, Validator { +export class JsFuncComponent implements OnInit, OnChanges, OnDestroy, ControlValueAccessor, Validator { @ViewChild('javascriptEditor', {static: true}) javascriptEditorElmRef: ElementRef; @@ -178,28 +180,11 @@ export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, } ngOnInit(): void { - if (this.functionTitle || this.label) { - this.hideBrackets = true; - } if (!this.resultType || this.resultType.length === 0) { this.resultType = 'nocheck'; } - if (this.functionArgs) { - this.functionArgs.forEach((functionArg) => { - if (this.functionArgsString.length > 0) { - this.functionArgsString += ', '; - } - this.functionArgsString += functionArg; - }); - } - if (this.functionTitle) { - this.functionLabel = `${this.functionTitle}: f(${this.functionArgsString})`; - } else if (this.label) { - this.functionLabel = this.label; - } else { - this.functionLabel = - `function ${this.functionName ? this.functionName : ''}(${this.functionArgsString})${this.hideBrackets ? '' : ' {'}`; - } + this.updateFunctionArgsString() + this.updateFunctionLabel(); const editorElement = this.javascriptEditorElmRef.nativeElement; let editorOptions: Partial = { mode: 'ace/mode/javascript', @@ -251,21 +236,7 @@ export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, } }); } - // @ts-ignore - if (!!this.highlightRules && !!this.jsEditor.session.$mode) { - // @ts-ignore - const newMode = new this.jsEditor.session.$mode.constructor(); - newMode.$highlightRules = new newMode.HighlightRules(); - for(const group in this.highlightRules) { - if(!!newMode.$highlightRules.$rules[group]) { - newMode.$highlightRules.$rules[group].unshift(...this.highlightRules[group]); - } else { - newMode.$highlightRules.$rules[group] = this.highlightRules[group]; - } - } - // @ts-ignore - this.jsEditor.session.$onChangeMode(newMode); - } + this.updateHighlightRules(); this.updateJsWorkerGlobals(); this.initialCompleters = this.jsEditor.completers || []; this.updateCompleters(); @@ -277,6 +248,16 @@ export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, ); } + ngOnChanges(changes: SimpleChanges): void { + for (const propName of Object.keys(changes)) { + const { firstChange, currentValue, previousValue } = changes[propName]; + const isChanged = isObject(currentValue) ? !isEqual(currentValue, previousValue) : currentValue !== previousValue; + if (!firstChange && isChanged) { + this.updateByChangesPropName(propName); + } + } + } + ngOnDestroy(): void { if (this.editorResize$) { this.editorResize$.disconnect(); @@ -329,6 +310,32 @@ export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, ); } + private updateFunctionArgsString(): void { + this.functionArgsString = ''; + if (this.functionArgs) { + this.functionArgsString = this.functionArgs.join(', '); + } + } + + private updateFunctionLabel(): void { + if (this.functionTitle || this.label) { + this.hideBrackets = true; + } + if (this.functionTitle) { + this.functionLabel = `${this.functionTitle}: f(${this.functionArgsString})`; + } else if (this.label) { + this.functionLabel = this.label; + } else { + this.functionLabel = + `function ${this.functionName ? this.functionName : ''}(${this.functionArgsString})${this.hideBrackets ? '' : ' {'}`; + } + this.cd.markForCheck(); + } + + private updatedScriptLanguage() { + this.jsEditor.session.setMode(`ace/mode/${ScriptLanguage.TBEL === this.scriptLanguage ? 'tbel' : 'javascript'}`); + } + validateOnSubmit(): Observable { if (!this.disabled) { this.cleanupJsErrors(); @@ -539,6 +546,56 @@ export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, } } + private updateByChangesPropName(propName: string): void { + switch (propName) { + case 'functionArgs': + this.updateFunctionArgsString() + this.updateFunctionLabel(); + this.updateJsWorkerGlobals(); + break; + case 'label': + case 'functionTitle': + case 'functionName': + this.updateFunctionLabel(); + break; + case 'scriptLanguage': + this.updatedScriptLanguage(); + break; + case 'disableUndefinedCheck': + case 'globalVariables': + this.updateJsWorkerGlobals(); + break; + case 'editorCompleter': + this.updateCompleters(); + break; + case 'highlightRules': + this.updateHighlightRules(); + break; + } + } + + private updateHighlightRules(): void { + // @ts-ignore + if (!!this.highlightRules && !!this.jsEditor.session.$mode) { + // @ts-ignore + const newMode = new this.jsEditor.session.$mode.constructor(); + newMode.$highlightRules = new newMode.HighlightRules(); + for(const group in this.highlightRules) { + if(!!newMode.$highlightRules.$rules[group]) { + newMode.$highlightRules.$rules[group].unshift(...this.highlightRules[group]); + } else { + newMode.$highlightRules.$rules[group] = this.highlightRules[group]; + } + } + const identifierRule = newMode.$highlightRules.$rules.no_regex.find(rule => rule.token?.includes('identifier')); + if (identifierRule) { + identifierRule.next = 'start'; + } + // @ts-ignore + this.jsEditor.session.$onChangeMode(newMode); + } + } + private updateJsWorkerGlobals() { // @ts-ignore if (!!this.jsEditor.session.$worker) { diff --git a/ui-ngx/src/app/shared/components/image/images-in-use-dialog.component.html b/ui-ngx/src/app/shared/components/resource/resources-in-use-dialog.component.html similarity index 71% rename from ui-ngx/src/app/shared/components/image/images-in-use-dialog.component.html rename to ui-ngx/src/app/shared/components/resource/resources-in-use-dialog.component.html index 4f4667ceb8..08a35aa412 100644 --- a/ui-ngx/src/app/shared/components/image/images-in-use-dialog.component.html +++ b/ui-ngx/src/app/shared/components/resource/resources-in-use-dialog.component.html @@ -15,10 +15,10 @@ limitations under the License. --> -

    {{title}}

    -
    -
    - +

    {{configuration.title}}

    +
    +
    +
    @@ -28,14 +28,14 @@ [indeterminate]="dataSource.selection.hasValue() && (dataSource.isAllSelected() | async) === false"> - + + (change)="$event ? dataSource.selection.toggle(resource) : null" + [checked]="dataSource.selection.isSelected(resource)"> - + {{ image.title }} @@ -43,30 +43,30 @@ - - {{ 'image.name' | translate }} + + {{ 'resource.title' | translate }} - - {{ translate.get('image.selected-images', {count: dataSource.selection.selected.length}) | async }} + + {{ translate.get(configuration.selectedText, {count: dataSource.selection.selected.length}) | async }} - - {{ image.title }} + + {{ resource.title }} - + + (click)="toggleShowReferences($event, resource, showReferencesButton)">{{ 'image.references' | translate }} - + + [class.mat-selected]="dataSource.selection.isSelected(resource)" + *matRowDef="let resource; columns: configuration.columns;">
    @@ -76,7 +76,7 @@
    - + -
    +
    diff --git a/ui-ngx/src/app/shared/components/image/images-in-use-dialog.component.scss b/ui-ngx/src/app/shared/components/resource/resources-in-use-dialog.component.scss similarity index 96% rename from ui-ngx/src/app/shared/components/image/images-in-use-dialog.component.scss rename to ui-ngx/src/app/shared/components/resource/resources-in-use-dialog.component.scss index 5c053e984f..380a6070eb 100644 --- a/ui-ngx/src/app/shared/components/image/images-in-use-dialog.component.scss +++ b/ui-ngx/src/app/shared/components/resource/resources-in-use-dialog.component.scss @@ -14,7 +14,7 @@ * limitations under the License. */ :host { - .tb-images-in-use-content { + .tb-resources-in-use-content { display: flex; flex-direction: column; gap: 24px; diff --git a/ui-ngx/src/app/shared/components/image/images-in-use-dialog.component.ts b/ui-ngx/src/app/shared/components/resource/resources-in-use-dialog.component.ts similarity index 66% rename from ui-ngx/src/app/shared/components/image/images-in-use-dialog.component.ts rename to ui-ngx/src/app/shared/components/resource/resources-in-use-dialog.component.ts index 681e9a2c53..6f83fded7d 100644 --- a/ui-ngx/src/app/shared/components/image/images-in-use-dialog.component.ts +++ b/ui-ngx/src/app/shared/components/resource/resources-in-use-dialog.component.ts @@ -20,37 +20,50 @@ import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { DialogComponent } from '@shared/components/dialog.component'; import { Router } from '@angular/router'; -import { ImageReferences, ImageResourceInfo, ImageResourceInfoWithReferences } from '@shared/models/resource.models'; -import { ImagesDatasource } from '@shared/components/image/images-datasource'; +import { + ResourceReferences, + ResourceInfoWithReferences, + ResourceInfo +} from '@shared/models/resource.models'; import { MatButton } from '@angular/material/button'; import { TbPopoverService } from '@shared/components/popover.service'; import { ImageReferencesComponent } from '@shared/components/image/image-references.component'; import { TranslateService } from '@ngx-translate/core'; +import { Datasource } from "@shared/models/widget.models"; -export interface ImagesInUseDialogData { +interface ResourcesInUseDialogDataConfiguration { + title: string; + message: string; + columns: string[]; + deleteText: string; + selectedText: string; + datasource?: Datasource; +} + +export interface ResourcesInUseDialogData { multiple: boolean; - images: ImageResourceInfoWithReferences[]; + resources: ResourceInfoWithReferences[]; + configuration: ResourcesInUseDialogDataConfiguration; } @Component({ - selector: 'tb-images-in-use-dialog', - templateUrl: './images-in-use-dialog.component.html', - styleUrls: ['./images-in-use-dialog.component.scss'] + selector: 'tb-resources-in-use-dialog', + templateUrl: './resources-in-use-dialog.component.html', + styleUrls: ['./resources-in-use-dialog.component.scss'] }) -export class ImagesInUseDialogComponent extends - DialogComponent implements OnInit { - - title: string; - message: string; +export class ResourcesInUseDialogComponent extends + DialogComponent implements OnInit { - references: ImageReferences; + displayPreview: boolean; + configuration: ResourcesInUseDialogDataConfiguration; + references: ResourceReferences; - dataSource: ImagesDatasource; + dataSource: Datasource; constructor(protected store: Store, protected router: Router, - @Inject(MAT_DIALOG_DATA) public data: ImagesInUseDialogData, - public dialogRef: MatDialogRef, + @Inject(MAT_DIALOG_DATA) public data: ResourcesInUseDialogData, + public dialogRef: MatDialogRef, public translate: TranslateService, private renderer: Renderer2, private viewContainerRef: ViewContainerRef, @@ -59,14 +72,12 @@ export class ImagesInUseDialogComponent extends } ngOnInit(): void { + this.configuration = this.data.configuration; + this.displayPreview = this.data.configuration.columns.includes('preview'); if (this.data.multiple) { - this.title = this.translate.instant('image.images-are-in-use'); - this.message = this.translate.instant('image.images-are-in-use-text'); - this.dataSource = new ImagesDatasource(null, this.data.images, entity => true); + this.dataSource = this.data.configuration.datasource; } else { - this.title = this.translate.instant('image.image-is-in-use'); - this.message = this.translate.instant('image.image-is-in-use-text', {title: this.data.images[0].title}); - this.references = this.data.images[0].references; + this.references = this.data.resources[0].references; } } @@ -78,11 +89,11 @@ export class ImagesInUseDialogComponent extends if (this.data.multiple) { this.dialogRef.close(this.dataSource.selection.selected); } else { - this.dialogRef.close(this.data.images); + this.dialogRef.close(this.data.resources); } } - toggleShowReferences($event: Event, image: ImageResourceInfoWithReferences, referencesButton: MatButton) { + toggleShowReferences($event: Event, resource: ResourceInfoWithReferences, referencesButton: MatButton) { if ($event) { $event.stopPropagation(); } @@ -93,7 +104,7 @@ export class ImagesInUseDialogComponent extends const referencesPopover = this.popoverService.displayPopover(trigger, this.renderer, this.viewContainerRef, ImageReferencesComponent, 'top', true, null, { - references: image.references + references: resource.references }, {}, {}, {}, false, visible => { diff --git a/ui-ngx/src/app/shared/components/value-input.component.html b/ui-ngx/src/app/shared/components/value-input.component.html index e8409d869e..1664a4af95 100644 --- a/ui-ngx/src/app/shared/components/value-input.component.html +++ b/ui-ngx/src/app/shared/components/value-input.component.html @@ -32,8 +32,8 @@ - + - + - +
    - + -
    diff --git a/ui-ngx/src/app/shared/components/value-input.component.ts b/ui-ngx/src/app/shared/components/value-input.component.ts index 312e12a2a7..07bdaaa742 100644 --- a/ui-ngx/src/app/shared/components/value-input.component.ts +++ b/ui-ngx/src/app/shared/components/value-input.component.ts @@ -81,6 +81,14 @@ export class ValueInputComponent implements OnInit, OnDestroy, OnChanges, Contro @coerceBoolean() shortBooleanField = false; + @Input() + @coerceBoolean() + required = true; + + @Input() + @coerceBoolean() + hideJsonEdit = false; + @Input() layout: ValueInputLayout | Layout = 'row'; diff --git a/ui-ngx/src/app/shared/import-export/import-export.service.ts b/ui-ngx/src/app/shared/import-export/import-export.service.ts index 5212c15fd4..022a844ceb 100644 --- a/ui-ngx/src/app/shared/import-export/import-export.service.ts +++ b/ui-ngx/src/app/shared/import-export/import-export.service.ts @@ -88,6 +88,8 @@ import { ExportResourceDialogDialogResult } from '@shared/import-export/export-resource-dialog.component'; import { FormProperty, propertyValid } from '@shared/models/dynamic-form.models'; +import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; +import { CalculatedField } from '@shared/models/calculated-field.models'; export type editMissingAliasesFunction = (widgets: Array, isSingleWidget: boolean, customTitle: string, missingEntityAliases: EntityAliases) => Observable; @@ -116,6 +118,7 @@ export class ImportExportService { private imageService: ImageService, private utils: UtilsService, private itembuffer: ItemBufferService, + private calculatedFieldsService: CalculatedFieldsService, private dialog: MatDialog) { } @@ -171,6 +174,35 @@ export class ImportExportService { ); } + public exportCalculatedField(calculatedFieldId: string): void { + this.calculatedFieldsService.getCalculatedFieldById(calculatedFieldId).subscribe({ + next: (calculatedField) => { + let name = calculatedField.name; + name = name.toLowerCase().replace(/\W/g, '_'); + this.exportToPc(this.prepareCalculatedFieldExport(calculatedField), name); + }, + error: (e) => { + this.handleExportError(e, 'calculated-fields.export-failed-error'); + } + }); + } + + public importCalculatedField(entityId: EntityId): Observable { + return this.openImportDialog('calculated-fields.import', 'calculated-fields.file').pipe( + mergeMap((calculatedField: CalculatedField) => { + if (!this.validateImportedCalculatedField({ entityId, ...calculatedField })) { + this.store.dispatch(new ActionNotificationShow( + {message: this.translate.instant('calculated-fields.invalid-file-error'), + type: 'error'})); + throw new Error('Invalid calculated field file'); + } else { + return this.calculatedFieldsService.saveCalculatedField(this.prepareImport({ entityId, ...calculatedField })); + } + }), + catchError(() => of(null)), + ); + } + public exportDashboard(dashboardId: string) { this.getIncludeResourcesPreference('includeResourcesInExportDashboard').subscribe(includeResources => { this.openExportDialog('dashboard.export', 'dashboard.export-prompt', includeResources).subscribe(result => { @@ -957,6 +989,16 @@ export class ImportExportService { } } + private validateImportedCalculatedField(calculatedField: CalculatedField): boolean { + const { name, configuration, entityId } = calculatedField; + return isNotEmptyStr(name) + && isDefined(configuration) + && isDefined(entityId?.id) + && !!Object.keys(configuration.arguments).length + && isDefined(configuration.expression) + && isDefined(configuration.output) + } + private validateImportedImage(image: ImageExportData): boolean { return !(!isNotEmptyStr(image.data) || !isNotEmptyStr(image.title) @@ -1209,6 +1251,11 @@ export class ImportExportService { return profile; } + private prepareCalculatedFieldExport(calculatedField: CalculatedField): CalculatedField { + delete calculatedField.entityId; + return this.prepareExport(calculatedField); + } + private prepareExport(data: any): any { const exportedData = deepClone(data); if (isDefined(exportedData.id)) { diff --git a/ui-ngx/src/app/shared/models/ace/ace.models.ts b/ui-ngx/src/app/shared/models/ace/ace.models.ts index f6af2d6a52..49cf670989 100644 --- a/ui-ngx/src/app/shared/models/ace/ace.models.ts +++ b/ui-ngx/src/app/shared/models/ace/ace.models.ts @@ -365,5 +365,15 @@ export interface AceHighlightRule { next?: string; } +export const dotOperatorHighlightRule: AceHighlightRule = { + token: 'punctuation.operator', + regex: /[.](?![.])/, +}; + +export const endGroupHighlightRule: AceHighlightRule = { + regex: '', + token: 'empty', + next: 'no_regex' +}; diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts new file mode 100644 index 0000000000..c898342c68 --- /dev/null +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -0,0 +1,669 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + AdditionalDebugActionConfig, + HasEntityDebugSettings, + HasTenantId, + HasVersion +} from '@shared/models/entity.models'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; +import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; +import { EntityId } from '@shared/models/id/entity-id'; +import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; +import { EntityType } from '@shared/models/entity-type.models'; +import { AliasFilterType } from '@shared/models/alias.models'; +import { Observable } from 'rxjs'; +import { TbEditorCompleter } from '@shared/models/ace/completion.models'; +import { + AceHighlightRule, + AceHighlightRules, + dotOperatorHighlightRule, + endGroupHighlightRule +} from '@shared/models/ace/ace.models'; + +export interface CalculatedField extends Omit, 'label'>, HasVersion, HasEntityDebugSettings, HasTenantId, ExportableEntity { + configuration: CalculatedFieldConfiguration; + type: CalculatedFieldType; + entityId: EntityId; +} + +export enum CalculatedFieldType { + SIMPLE = 'SIMPLE', + SCRIPT = 'SCRIPT', +} + +export const CalculatedFieldTypeTranslations = new Map( + [ + [CalculatedFieldType.SIMPLE, 'calculated-fields.type.simple'], + [CalculatedFieldType.SCRIPT, 'calculated-fields.type.script'], + ] +) + +export interface CalculatedFieldConfiguration { + type: CalculatedFieldType; + expression: string; + arguments: Record; + output: CalculatedFieldOutput; +} + +export interface CalculatedFieldOutput { + type: OutputType; + name: string; + scope?: AttributeScope; + decimalsByDefault?: number; +} + +export enum ArgumentEntityType { + Current = 'CURRENT', + Device = 'DEVICE', + Asset = 'ASSET', + Customer = 'CUSTOMER', + Tenant = 'TENANT', +} + +export const ArgumentEntityTypeTranslations = new Map( + [ + [ArgumentEntityType.Current, 'calculated-fields.argument-current'], + [ArgumentEntityType.Device, 'calculated-fields.argument-device'], + [ArgumentEntityType.Asset, 'calculated-fields.argument-asset'], + [ArgumentEntityType.Customer, 'calculated-fields.argument-customer'], + [ArgumentEntityType.Tenant, 'calculated-fields.argument-tenant'], + ] +) + +export enum ArgumentType { + Attribute = 'ATTRIBUTE', + LatestTelemetry = 'TS_LATEST', + Rolling = 'TS_ROLLING', +} + +export enum TestArgumentType { + Single = 'SINGLE_VALUE', + Rolling = 'TS_ROLLING', +} + +export const TestArgumentTypeMap = new Map( + [ + [ArgumentType.Attribute, TestArgumentType.Single], + [ArgumentType.LatestTelemetry, TestArgumentType.Single], + [ArgumentType.Rolling, TestArgumentType.Rolling], + ] +) + +export enum OutputType { + Attribute = 'ATTRIBUTES', + Timeseries = 'TIME_SERIES', +} + +export const OutputTypeTranslations = new Map( + [ + [OutputType.Attribute, 'calculated-fields.attribute'], + [OutputType.Timeseries, 'calculated-fields.timeseries'], + ] +) + +export const ArgumentTypeTranslations = new Map( + [ + [ArgumentType.Attribute, 'calculated-fields.attribute'], + [ArgumentType.LatestTelemetry, 'calculated-fields.latest-telemetry'], + [ArgumentType.Rolling, 'calculated-fields.rolling'], + ] +) + +export interface CalculatedFieldArgument { + refEntityKey: RefEntityKey; + defaultValue?: string; + refEntityId?: RefEntityKey; + limit?: number; + timeWindow?: number; +} + +export interface RefEntityKey { + key: string; + type: ArgumentType; + scope?: AttributeScope; +} + +export interface RefEntityKey { + entityType: ArgumentEntityType; + id: string; +} + +export interface CalculatedFieldArgumentValue extends CalculatedFieldArgument { + argumentName: string; +} + +export type CalculatedFieldTestScriptFn = (calculatedField: CalculatedField, argumentsObj?: Record, closeAllOnSave?: boolean) => Observable; + +export interface CalculatedFieldDialogData { + value?: CalculatedField; + buttonTitle: string; + entityId: EntityId; + debugLimitsConfiguration: string; + tenantId: string; + entityName?: string; + additionalDebugActionConfig: AdditionalDebugActionConfig<(calculatedField: CalculatedField) => void>; + getTestScriptDialogFn: CalculatedFieldTestScriptFn; + isDirty?: boolean; +} + +export interface CalculatedFieldDebugDialogData { + tenantId: string; + value: CalculatedField; + getTestScriptDialogFn: CalculatedFieldTestScriptFn; +} + +export interface CalculatedFieldTestScriptInputParams { + arguments: CalculatedFieldEventArguments; + expression: string; +} + +export interface CalculatedFieldTestScriptDialogData extends CalculatedFieldTestScriptInputParams { + argumentsEditorCompleter: TbEditorCompleter; + argumentsHighlightRules: AceHighlightRules; + openCalculatedFieldEdit?: boolean; +} + +export interface ArgumentEntityTypeParams { + title: string; + entityType: EntityType +} + +export const ArgumentEntityTypeParamsMap =new Map([ + [ArgumentEntityType.Device, { title: 'calculated-fields.device-name', entityType: EntityType.DEVICE }], + [ArgumentEntityType.Asset, { title: 'calculated-fields.asset-name', entityType: EntityType.ASSET }], + [ArgumentEntityType.Customer, { title: 'calculated-fields.customer-name', entityType: EntityType.CUSTOMER }], +]) + +export const getCalculatedFieldCurrentEntityFilter = (entityName: string, entityId: EntityId) => { + switch (entityId.entityType) { + case EntityType.ASSET_PROFILE: + return { + assetTypes: [entityName], + type: AliasFilterType.assetType + }; + case EntityType.DEVICE_PROFILE: + return { + deviceTypes: [entityName], + type: AliasFilterType.deviceType + }; + default: + return { + type: AliasFilterType.singleEntity, + singleEntity: entityId, + }; + } +} + +export interface CalculatedFieldArgumentValueBase { + argumentName: string; + type: ArgumentType; +} + +export interface CalculatedFieldAttributeArgumentValue extends CalculatedFieldArgumentValueBase { + ts: number; + value: ValueType; +} + +export interface CalculatedFieldLatestTelemetryArgumentValue extends CalculatedFieldArgumentValueBase { + ts: number; + value: ValueType; +} + +export interface CalculatedFieldRollingTelemetryArgumentValue extends CalculatedFieldArgumentValueBase { + timeWindow: { startTs: number; endTs: number; }; + values: CalculatedFieldSingleArgumentValue[]; +} + +export type CalculatedFieldSingleArgumentValue = CalculatedFieldAttributeArgumentValue & CalculatedFieldLatestTelemetryArgumentValue; + +export type CalculatedFieldArgumentEventValue = CalculatedFieldAttributeArgumentValue | CalculatedFieldLatestTelemetryArgumentValue | CalculatedFieldRollingTelemetryArgumentValue; + +export type CalculatedFieldEventArguments = Record>; + +export const CalculatedFieldCtxLatestTelemetryArgumentAutocomplete = { + meta: 'object', + type: '{ ts: number; value: any; }', + description: 'Calculated field context latest telemetry value argument.', + children: { + ts: { + meta: 'number', + type: 'number', + description: 'Time stamp', + }, + value: { + meta: 'any', + type: 'any', + description: 'Value', + } + }, +}; + +export const CalculatedFieldCtxAttributeValueArgumentAutocomplete = { + meta: 'object', + type: '{ ts: number; value: any; }', + description: 'Calculated field context attribute value argument.', + children: { + ts: { + meta: 'number', + type: 'number', + description: 'Time stamp', + }, + value: { + meta: 'any', + type: 'any', + description: 'Value', + } + }, +}; + +export const CalculatedFieldLatestTelemetryArgumentAutocomplete = { + meta: 'any', + type: 'any', + description: 'Calculated field latest telemetry argument value.', +}; + +export const CalculatedFieldAttributeValueArgumentAutocomplete = { + meta: 'any', + type: 'any', + description: 'Calculated field attribute argument value.', +}; + +export const CalculatedFieldRollingValueArgumentFunctionsAutocomplete = { + max: { + meta: 'function', + description: 'Returns the maximum value of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + args: [ + { + name: 'ignoreNaN', + description: 'Whether to ignore NaN values. Equals true by default.', + type: 'boolean', + optional: true, + } + ], + return: { + description: 'The maximum value, or NaN if applicable', + type: 'number' + } + }, + min: { + meta: 'function', + description: 'Returns the minimum value of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + args: [ + { + name: 'ignoreNaN', + description: 'Whether to ignore NaN values. Equals true by default.', + type: 'boolean', + optional: true, + } + ], + return: { + description: 'The minimum value, or NaN if applicable', + type: 'number' + } + }, + mean: { + meta: 'function', + description: 'Computes the mean value of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + args: [ + { + name: 'ignoreNaN', + description: 'Whether to ignore NaN values. Equals true by default.', + type: 'boolean', + optional: true, + } + ], + return: { + description: 'The mean value, or NaN if applicable', + type: 'number' + } + }, + avg: { + meta: 'function', + description: 'Computes the average value of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + args: [ + { + name: 'ignoreNaN', + description: 'Whether to ignore NaN values. Equals true by default.', + type: 'boolean', + optional: true, + } + ], + return: { + description: 'The average value, or NaN if applicable', + type: 'number' + } + }, + std: { + meta: 'function', + description: 'Computes the standard deviation of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + args: [ + { + name: 'ignoreNaN', + description: 'Whether to ignore NaN values. Equals true by default.', + type: 'boolean', + optional: true, + } + ], + return: { + description: 'The standard deviation, or NaN if applicable', + type: 'number' + } + }, + median: { + meta: 'function', + description: 'Computes the median value of the rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + args: [ + { + name: 'ignoreNaN', + description: 'Whether to ignore NaN values. Equals true by default.', + type: 'boolean', + optional: true, + } + ], + return: { + description: 'The median value, or NaN if applicable', + type: 'number' + } + }, + count: { + meta: 'function', + description: 'Counts values of the rolling argument. Counts non-NaN values if ignoreNaN is true, otherwise - total size.', + args: [ + { + name: 'ignoreNaN', + description: 'Whether to ignore NaN values. Equals true by default.', + type: 'boolean', + optional: true, + } + ], + return: { + description: 'The count of values', + type: 'number' + } + }, + last: { + meta: 'function', + description: 'Returns the last non-NaN value of the rolling argument values if ignoreNaN is true, otherwise - the last value.', + args: [ + { + name: 'ignoreNaN', + description: 'Whether to ignore NaN values. Equals true by default.', + type: 'boolean', + optional: true, + } + ], + return: { + description: 'The last value, or NaN if applicable', + type: 'number' + } + }, + first: { + meta: 'function', + description: 'Returns the first non-NaN value of the rolling argument values if ignoreNaN is true, otherwise - the first value.', + args: [ + { + name: 'ignoreNaN', + description: 'Whether to ignore NaN values. Equals true by default.', + type: 'boolean', + optional: true, + } + ], + return: { + description: 'The first value, or NaN if applicable', + type: 'number' + } + }, + sum: { + meta: 'function', + description: 'Computes the sum of rolling argument values. Returns NaN if any value is NaN and ignoreNaN is false.', + args: [ + { + name: 'ignoreNaN', + description: 'Whether to ignore NaN values. Equals true by default.', + type: 'boolean', + optional: true, + } + ], + return: { + description: 'The sum of values, or NaN if applicable', + type: 'number' + } + }, + merge: { + meta: 'function', + description: 'Merges current object with other time series rolling argument into a single object by aligning their timestamped values. Supports optional configurable settings.', + args: [ + { + name: 'other', + description: "A time series rolling argument to be merged with the current object.", + type: "object", + optional: true + }, + { + name: "settings", + description: "Optional settings controlling the merging process. Supported keys: 'ignoreNaN' (boolean, equals true by default) to determine whether NaN values should be ignored; 'timeWindow' (object, empty by default) to apply time window filtering.", + type: "object", + optional: true + } + ], + return: { + description: 'A new object containing merged timestamped values from all provided arguments, aligned based on timestamps and filtered according to settings.', + type: '{ values: { ts: number; values: number[]; }[]; timeWindow: { startTs: number; endTs: number } }; }', + } + }, + mergeAll: { + meta: 'function', + description: 'Merges current object with other time series rolling arguments into a single object by aligning their timestamped values. Supports optional configurable settings.', + args: [ + { + name: 'others', + description: "A list of time series rolling arguments to be merged with the current object.", + type: "object[]", + optional: true + }, + { + name: "settings", + description: "Optional settings controlling the merging process. Supported keys: 'ignoreNaN' (boolean, equals true by default) to determine whether NaN values should be ignored; 'timeWindow' (object, empty by default) to apply time window filtering.", + type: "object", + optional: true + } + ], + return: { + description: 'A new object containing merged timestamped values from all provided arguments, aligned based on timestamps and filtered according to settings.', + type: '{ values: { ts: number; values: number[]; }[]; timeWindow: { startTs: number; endTs: number } }; }', + } + } +}; + +export const CalculatedFieldRollingValueArgumentAutocomplete = { + meta: 'object', + type: '{ values: { ts: number; value: number; }[]; timeWindow: { startTs: number; endTs: number } }; }', + description: 'Calculated field rolling value argument.', + children: { + ...CalculatedFieldRollingValueArgumentFunctionsAutocomplete, + values: { + meta: 'array', + type: '{ ts: number; value: any; }[]', + description: 'Values array', + }, + timeWindow: { + meta: 'object', + type: '{ startTs: number; endTs: number }', + description: 'Time window configuration', + children: { + startTs: { + meta: 'number', + type: 'number', + description: 'Start time stamp', + }, + endTs: { + meta: 'number', + type: 'number', + description: 'End time stamp', + } + } + } + }, +}; + +export const getCalculatedFieldArgumentsEditorCompleter = (argumentsObj: Record): TbEditorCompleter => { + return new TbEditorCompleter(Object.keys(argumentsObj).reduce((acc, key) => { + switch (argumentsObj[key].refEntityKey.type) { + case ArgumentType.Attribute: + acc[key] = CalculatedFieldAttributeValueArgumentAutocomplete; + acc.ctx.children.args.children[key] = CalculatedFieldCtxAttributeValueArgumentAutocomplete; + break; + case ArgumentType.LatestTelemetry: + acc[key] = CalculatedFieldLatestTelemetryArgumentAutocomplete; + acc.ctx.children.args.children[key] = CalculatedFieldCtxLatestTelemetryArgumentAutocomplete; + break; + case ArgumentType.Rolling: + acc[key] = CalculatedFieldRollingValueArgumentAutocomplete; + acc.ctx.children.args.children[key] = CalculatedFieldRollingValueArgumentAutocomplete; + break; + } + return acc; + }, { + ctx: { + meta: 'object', + type: '{ args: { [key: string]: object } }', + description: 'Calculated field context.', + children: { + args: { + meta: 'object', + type: '{ [key: string]: object }', + description: 'Calculated field context arguments.', + children: {} + } + } + } + })); +} + +export const getCalculatedFieldArgumentsHighlights = ( + argumentsObj: Record +): AceHighlightRules => { + const calculatedFieldArgumentsKeys = Object.keys(argumentsObj).map(key => ({ + token: 'tb.calculated-field-key', + regex: `\\b${key}\\b`, + next: argumentsObj[key].refEntityKey.type === ArgumentType.Rolling + ? 'calculatedFieldRollingArgumentValue' + : 'start' + })); + const calculatedFieldCtxArgumentsHighlightRules = { + calculatedFieldCtxArgs: [ + dotOperatorHighlightRule, + ...calculatedFieldArgumentsKeys.map(argumentRule => argumentRule.next === 'start' ? {...argumentRule, next: 'calculatedFieldSingleArgumentValue' } : argumentRule), + endGroupHighlightRule + ] + }; + + return { + start: [ + calculatedFieldArgumentsContextHighlightRules, + ...calculatedFieldArgumentsKeys, + ], + ...calculatedFieldArgumentsContextValueHighlightRules, + ...calculatedFieldCtxArgumentsHighlightRules, + ...calculatedFieldSingleArgumentValueHighlightRules, + ...calculatedFieldRollingArgumentValueHighlightRules, + ...calculatedFieldTimeWindowArgumentValueHighlightRules + }; +}; + +const calculatedFieldArgumentsContextHighlightRules: AceHighlightRule = { + token: 'tb.calculated-field-ctx', + regex: /ctx/, + next: 'calculatedFieldCtxValue' +} + +const calculatedFieldArgumentsContextValueHighlightRules: AceHighlightRules = { + calculatedFieldCtxValue: [ + dotOperatorHighlightRule, + { + token: 'tb.calculated-field-args', + regex: /args/, + next: 'calculatedFieldCtxArgs' + }, + endGroupHighlightRule + ] +} + +const calculatedFieldSingleArgumentValueHighlightRules: AceHighlightRules = { + calculatedFieldSingleArgumentValue: [ + dotOperatorHighlightRule, + { + token: 'tb.calculated-field-value', + regex: /value/, + next: 'no_regex' + }, + { + token: 'tb.calculated-field-ts', + regex: /ts/, + next: 'no_regex' + }, + endGroupHighlightRule + ], +} + +const calculatedFieldRollingArgumentValueFunctionsHighlightRules: Array = + ['max', 'min', 'avg', 'mean', 'std', 'median', 'count', 'last', 'first', 'sum', 'merge', 'mergeAll'].map(funcName => ({ + token: 'tb.calculated-field-func', + regex: `\\b${funcName}\\b`, + next: 'no_regex' + })); + +const calculatedFieldRollingArgumentValueHighlightRules: AceHighlightRules = { + calculatedFieldRollingArgumentValue: [ + dotOperatorHighlightRule, + { + token: 'tb.calculated-field-values', + regex: /values/, + next: 'no_regex' + }, + { + token: 'tb.calculated-field-time-window', + regex: /timeWindow/, + next: 'calculatedFieldRollingArgumentTimeWindow' + }, + ...calculatedFieldRollingArgumentValueFunctionsHighlightRules, + endGroupHighlightRule + ], +} + +const calculatedFieldTimeWindowArgumentValueHighlightRules: AceHighlightRules = { + calculatedFieldRollingArgumentTimeWindow: [ + dotOperatorHighlightRule, + { + token: 'tb.calculated-field-start-ts', + regex: /startTs/, + next: 'no_regex' + }, + { + token: 'tb.calculated-field-end-ts', + regex: /endTs/, + next: 'no_regex' + }, + endGroupHighlightRule + ] +} + +export const calculatedFieldDefaultScript = 'return {\n' + + ' // Convert Fahrenheit to Celsius\n' + + ' "temperatureCelsius": (temperature - 32) / 1.8\n' + + '};' diff --git a/ui-ngx/src/app/shared/models/constants.ts b/ui-ngx/src/app/shared/models/constants.ts index c811d11052..bdd61e060b 100644 --- a/ui-ngx/src/app/shared/models/constants.ts +++ b/ui-ngx/src/app/shared/models/constants.ts @@ -196,6 +196,7 @@ export const HelpLinks = { mobileApplication: `${helpBaseUrl}/docs${docPlatformPrefix}/mobile-center/applications/`, mobileBundle: `${helpBaseUrl}/docs${docPlatformPrefix}/mobile-center/mobile-center/`, mobileQrCode: `${helpBaseUrl}/docs${docPlatformPrefix}/user-guide/ui/mobile-qr-code/`, + calculatedField: `${helpBaseUrl}/docs${docPlatformPrefix}/`, } }; /* eslint-enable max-len */ diff --git a/ui-ngx/src/app/shared/models/entity-type.models.ts b/ui-ngx/src/app/shared/models/entity-type.models.ts index fe344352c3..48cad42e7b 100644 --- a/ui-ngx/src/app/shared/models/entity-type.models.ts +++ b/ui-ngx/src/app/shared/models/entity-type.models.ts @@ -49,7 +49,8 @@ export enum EntityType { OAUTH2_CLIENT = 'OAUTH2_CLIENT', DOMAIN = 'DOMAIN', MOBILE_APP_BUNDLE = 'MOBILE_APP_BUNDLE', - MOBILE_APP = 'MOBILE_APP' + MOBILE_APP = 'MOBILE_APP', + CALCULATED_FIELD = 'CALCULATED_FIELD', } export enum AliasEntityType { @@ -478,6 +479,18 @@ export const entityTypeTranslations = new Map void> { + action: Action; + title: string; +} + export type VersionedEntity = EntityInfoData & HasVersion | RuleChainMetaData; diff --git a/ui-ngx/src/app/shared/models/event.models.ts b/ui-ngx/src/app/shared/models/event.models.ts index 30f9a57b89..027fae1aa9 100644 --- a/ui-ngx/src/app/shared/models/event.models.ts +++ b/ui-ngx/src/app/shared/models/event.models.ts @@ -29,7 +29,8 @@ export enum EventType { export enum DebugEventType { DEBUG_RULE_NODE = 'DEBUG_RULE_NODE', - DEBUG_RULE_CHAIN = 'DEBUG_RULE_CHAIN' + DEBUG_RULE_CHAIN = 'DEBUG_RULE_CHAIN', + DEBUG_CALCULATED_FIELD = 'DEBUG_CALCULATED_FIELD' } export const eventTypeTranslations = new Map( @@ -39,6 +40,7 @@ export const eventTypeTranslations = new Map [EventType.STATS, 'event.type-stats'], [DebugEventType.DEBUG_RULE_NODE, 'event.type-debug-rule-node'], [DebugEventType.DEBUG_RULE_CHAIN, 'event.type-debug-rule-chain'], + [DebugEventType.DEBUG_CALCULATED_FIELD, 'event.type-debug-calculated-field'], ] ); @@ -80,7 +82,7 @@ export interface DebugRuleChainEventBody extends BaseEventBody { error?: string; } -export type EventBody = ErrorEventBody & LcEventEventBody & StatsEventBody & DebugRuleNodeEventBody & DebugRuleChainEventBody; +export type EventBody = ErrorEventBody & LcEventEventBody & StatsEventBody & DebugRuleNodeEventBody & DebugRuleChainEventBody & CalculatedFieldEventBody; export interface Event extends BaseData { tenantId: TenantId; @@ -90,6 +92,16 @@ export interface Event extends BaseData { body: EventBody; } +export interface CalculatedFieldEventBody extends BaseFilterEventBody { + calculatedFieldId: string; + entityId: string; + entityType: EntityType; + arguments: string, + result: string, + msgId: string; + msgType: string; +} + export interface BaseFilterEventBody { server?: string; } diff --git a/ui-ngx/src/app/shared/models/id/calculated-field-id.ts b/ui-ngx/src/app/shared/models/id/calculated-field-id.ts new file mode 100644 index 0000000000..f42e20f3d2 --- /dev/null +++ b/ui-ngx/src/app/shared/models/id/calculated-field-id.ts @@ -0,0 +1,26 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { EntityId } from './entity-id'; +import { EntityType } from '@shared/models/entity-type.models'; + +export class CalculatedFieldId implements EntityId { + entityType = EntityType.CALCULATED_FIELD; + id: string; + constructor(id: string) { + this.id = id; + } +} diff --git a/ui-ngx/src/app/shared/models/public-api.ts b/ui-ngx/src/app/shared/models/public-api.ts index 470b628800..53e4bb286e 100644 --- a/ui-ngx/src/app/shared/models/public-api.ts +++ b/ui-ngx/src/app/shared/models/public-api.ts @@ -61,3 +61,4 @@ export * from './widgets-bundle.model'; export * from './window-message.model'; export * from './usage.models'; export * from './query/query.models'; +export * from './regex.constants'; diff --git a/ui-ngx/src/app/shared/models/regex.constants.ts b/ui-ngx/src/app/shared/models/regex.constants.ts new file mode 100644 index 0000000000..b8b1be4e83 --- /dev/null +++ b/ui-ngx/src/app/shared/models/regex.constants.ts @@ -0,0 +1,21 @@ +/// +/// Copyright © 2016-2025 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +export const oneSpaceInsideRegex = /^\s*\S+(?:\s\S+)*\s*$/; + +export const charsWithNumRegex = /^[a-zA-Z_]+[a-zA-Z0-9_]*$/; + +export const digitsRegex = /^\d*$/; diff --git a/ui-ngx/src/app/shared/models/resource.models.ts b/ui-ngx/src/app/shared/models/resource.models.ts index eda0182383..0419e40a7c 100644 --- a/ui-ngx/src/app/shared/models/resource.models.ts +++ b/ui-ngx/src/app/shared/models/resource.models.ts @@ -120,28 +120,28 @@ export interface ImageExportData { export type ImageResourceType = 'tenant' | 'system'; export type TBResourceScope = 'tenant' | 'system'; -export type ImageReferences = {[entityType: string]: Array & HasTenantId>}; +export type ResourceReferences = {[entityType: string]: Array & HasTenantId>}; -export interface ImageResourceInfoWithReferences extends ImageResourceInfo { - references: ImageReferences; +export interface ResourceInfoWithReferences extends ResourceInfo { + references: ResourceReferences; } -export interface ImageDeleteResult { - image: ImageResourceInfo; +export interface ResourceDeleteResult { + resource: TbResourceInfo; success: boolean; - imageIsReferencedError?: boolean; + resourceIsReferencedError?: boolean; error?: any; - references?: ImageReferences; + references?: ResourceReferences; } -export const toImageDeleteResult = (image: ImageResourceInfo, e?: any): ImageDeleteResult => { +export const toResourceDeleteResult = (resource: ResourceInfo, e?: any): ResourceDeleteResult => { if (!e) { - return {image, success: true}; + return {resource, success: true}; } else { - const result: ImageDeleteResult = {image, success: false, error: e}; + const result: ResourceDeleteResult = {resource, success: false, error: e}; if (e?.status === 400 && e?.error?.success === false && e?.error?.references) { - const references: ImageReferences = e?.error?.references; - result.imageIsReferencedError = true; + const references: ResourceReferences = e?.error?.references; + result.resourceIsReferencedError = true; result.references = references; } return result; diff --git a/ui-ngx/src/app/shared/models/rule-node.models.ts b/ui-ngx/src/app/shared/models/rule-node.models.ts index ad7f3c01d5..32422a87e0 100644 --- a/ui-ngx/src/app/shared/models/rule-node.models.ts +++ b/ui-ngx/src/app/shared/models/rule-node.models.ts @@ -25,7 +25,7 @@ import { AfterViewInit, DestroyRef, Directive, EventEmitter, inject, OnInit } fr import { AbstractControl, UntypedFormGroup } from '@angular/forms'; import { RuleChainType } from '@shared/models/rule-chain.models'; import { DebugRuleNodeEventBody } from '@shared/models/event.models'; -import { HasEntityDebugSettings } from '@shared/models/entity.models'; +import { EntityTestScriptResult, HasEntityDebugSettings } from '@shared/models/entity.models'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; export interface RuleNodeConfiguration { @@ -372,10 +372,7 @@ export interface TestScriptInputParams { msgType: string; } -export interface TestScriptResult { - output: string; - error: string; -} +export type TestScriptResult = EntityTestScriptResult; export enum MessageType { POST_ATTRIBUTES_REQUEST = 'POST_ATTRIBUTES_REQUEST', diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 23c2d95762..23896141db 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -95,6 +95,13 @@ export interface DefaultTenantProfileConfiguration { rpcTtlDays: number; queueStatsTtlDays: number; ruleEngineExceptionsTtlDays: number; + + maxCalculatedFieldsPerEntity: number; + maxArgumentsPerCF: number; + maxDataPointsPerRollingArg: number; + maxStateSizeInKBytes: number; + maxSingleValueArgumentSizeInKBytes: number; + calculatedFieldDebugEventsRateLimit: string; } export type TenantProfileConfigurations = DefaultTenantProfileConfiguration; @@ -148,7 +155,13 @@ export function createTenantProfileConfiguration(type: TenantProfileType): Tenan alarmsTtlDays: 0, rpcTtlDays: 0, queueStatsTtlDays: 0, - ruleEngineExceptionsTtlDays: 0 + ruleEngineExceptionsTtlDays: 0, + maxCalculatedFieldsPerEntity: 5, + maxArgumentsPerCF: 10, + maxDataPointsPerRollingArg: 1000, + maxStateSizeInKBytes: 32, + maxSingleValueArgumentSizeInKBytes: 2, + calculatedFieldDebugEventsRateLimit: '' }; configuration = {...defaultConfiguration, type: TenantProfileType.DEFAULT}; break; diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index 6cc8de02eb..3795518ffc 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -49,6 +49,7 @@ export interface VersionCreateConfig { saveRelations: boolean; saveAttributes: boolean; saveCredentials: boolean; + saveCalculatedFields: boolean; } export enum VersionCreateRequestType { @@ -106,6 +107,7 @@ export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: syncStrategy: null, saveAttributes: !entityTypesWithoutRelatedData.has(entityType), saveRelations: !entityTypesWithoutRelatedData.has(entityType), + saveCalculatedFields: typesWithCalculatedFields.has(entityType), saveCredentials: true, allEntities: true, entityIds: [] @@ -118,6 +120,7 @@ export interface VersionLoadConfig { loadRelations: boolean; loadAttributes: boolean; loadCredentials: boolean; + loadCalculatedFields: boolean; } export enum VersionLoadRequestType { @@ -154,6 +157,7 @@ export function createDefaultEntityTypesVersionLoad(): {[entityType: string]: En loadAttributes: !entityTypesWithoutRelatedData.has(entityType), loadRelations: !entityTypesWithoutRelatedData.has(entityType), loadCredentials: true, + loadCalculatedFields: typesWithCalculatedFields.has(entityType), removeOtherEntities: false, findExistingEntityByName: true }; @@ -254,4 +258,7 @@ export interface EntityDataInfo { hasRelations: boolean; hasAttributes: boolean; hasCredentials: boolean; + hasCalculatedFields: boolean; } + +export const typesWithCalculatedFields = new Set([EntityType.DEVICE, EntityType.ASSET, EntityType.ASSET_PROFILE, EntityType.DEVICE_PROFILE]); diff --git a/ui-ngx/src/app/shared/shared.module.ts b/ui-ngx/src/app/shared/shared.module.ts index 9245673d89..7eb622029f 100644 --- a/ui-ngx/src/app/shared/shared.module.ts +++ b/ui-ngx/src/app/shared/shared.module.ts @@ -201,7 +201,7 @@ import { ImageGalleryComponent } from '@shared/components/image/image-gallery.co import { UploadImageDialogComponent } from '@shared/components/image/upload-image-dialog.component'; import { ImageDialogComponent } from '@shared/components/image/image-dialog.component'; import { ImageReferencesComponent } from '@shared/components/image/image-references.component'; -import { ImagesInUseDialogComponent } from '@shared/components/image/images-in-use-dialog.component'; +import { ResourcesInUseDialogComponent } from '@shared/components/resource/resources-in-use-dialog.component'; import { GalleryImageInputComponent } from '@shared/components/image/gallery-image-input.component'; import { MultipleGalleryImageInputComponent } from '@shared/components/image/multiple-gallery-image-input.component'; import { EmbedImageDialogComponent } from '@shared/components/image/embed-image-dialog.component'; @@ -224,6 +224,7 @@ import { IntervalOptionsConfigPanelComponent } from '@shared/components/time/int import { GroupingIntervalOptionsComponent } from '@shared/components/time/aggregation/grouping-interval-options.component'; import { JsFuncModulesComponent } from '@shared/components/js-func-modules.component'; import { JsFuncModuleRowComponent } from '@shared/components/js-func-module-row.component'; +import { EntityKeyAutocompleteComponent } from '@shared/components/entity/entity-key-autocomplete.component'; export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) { return markedOptionsService; @@ -425,14 +426,15 @@ export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) UploadImageDialogComponent, ImageDialogComponent, ImageReferencesComponent, - ImagesInUseDialogComponent, + ResourcesInUseDialogComponent, GalleryImageInputComponent, MultipleGalleryImageInputComponent, EmbedImageDialogComponent, ImageGalleryDialogComponent, WidgetButtonComponent, HexInputComponent, - ScadaSymbolInputComponent + ScadaSymbolInputComponent, + EntityKeyAutocompleteComponent, ], imports: [ CommonModule, @@ -688,13 +690,14 @@ export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) UploadImageDialogComponent, ImageDialogComponent, ImageReferencesComponent, - ImagesInUseDialogComponent, + ResourcesInUseDialogComponent, GalleryImageInputComponent, MultipleGalleryImageInputComponent, EmbedImageDialogComponent, ImageGalleryDialogComponent, WidgetButtonComponent, - ScadaSymbolInputComponent + ScadaSymbolInputComponent, + EntityKeyAutocompleteComponent, ] }) export class SharedModule { } diff --git a/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md b/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md new file mode 100644 index 0000000000..e35572bf97 --- /dev/null +++ b/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md @@ -0,0 +1,335 @@ +#### Calculated field TBEL script function + +The **calculate()** function is a user-defined script that allows you to perform custom calculations using [TBEL{:target="_blank"}](${siteBaseUrl}/docs${docPlatformPrefix}/user-guide/tbel/) on telemetry and attribute data. +It receives arguments configured in the calculated field setup and an additional `ctx` object, which provides access to all arguments. + +##### Function signature + +```javascript + function calculate(ctx, arg1, arg2, ...): object | object[] +``` + +##### Argument representation in the script + +Before describing how arguments are passed to the function, let's define how different argument types are **represented** inside the script. + +There are two types of arguments that can be used in the function: + +* single value arguments - represent the latest telemetry data or attribute. + ```json + { + "altitude": { + "ts": 1740644636669, + "value": 1034 + } + } + ``` + + * when accessed via `ctx.args`, they remain objects: + + ```javascript + var altitudeTimestamp = ctx.args.altitude.ts; + var altitudeValue = ctx.args.altitude.value; + ``` + + * when accessed as a **function parameter**, only the value is passed: + + ```javascript + function calculate(ctx, altitude/*(single value argument)*/, temperature/*(time series rolling argument)*/) { + // altitude = 1035 + } + ``` + +* time series rolling arguments - contain historical data within a defined time window. + ```json + { + "temperature": { + "timeWindow": { + "startTs": 1740643762896, + "endTs": 1740644662896 + }, + "values": [ + { "ts": 1740644355935, "value": 72.32 }, + { "ts": 1740644365935, "value": 72.86 }, + { "ts": 1740644375935, "value": 73.58 }, + { "ts": 1740644385935, "value": "NaN" } + ] + } + } + ``` + + * when accessed via `ctx.args`, they remain rolling argument objects: + + ```javascript + var startOfInterval = temperature.timeWindow.startTs; + var firstTimestamp = temperature.values[0].ts; + var firstValue = temperature.values[0].value; + ``` + + * when accessed as a **function parameter**, they are passed as rolling arguments, retaining their structure: + + ```javascript + function calculate(ctx, altitude/*(single value argument)*/, temperature/*(time series rolling argument)*/) { + var avgTemp = temperature.mean(); // Use rolling argument functions + } + ``` + + **Built-in methods for rolling arguments** + + Time series rolling arguments provide built-in functions for calculations. + These functions accept an optional `ignoreNaN` boolean parameter, which controls how NaN values are handled. + Each function has two function signatures: + + * **Without parameters:** `method()` → called **without parameters** and defaults to `ignoreNaN = true`, meaning NaN values are ignored. + * **With an explicit parameter:** `method(boolean ignoreNaN)` → called with a boolean `ignoreNaN` parameter: + * `true` → ignores NaN values (default behavior). + * `false` → includes NaN values in calculations. + + | Method | Default Behavior (`ignoreNaN = true`) | Alternative (`ignoreNaN = false`) | + |------------|--------------------------------------------------|---------------------------------------------| + | `max()` | Returns the highest value, ignoring NaN values. | Returns NaN if any NaN values exist. | + | `min()` | Returns the lowest value, ignoring NaN values. | Returns NaN if any NaN values exist. | + | `mean()` | Computes the average value, ignoring NaN values. | Returns NaN if any NaN values exist. | + | `std()` | Calculates the standard deviation, ignoring NaN. | Returns NaN if any NaN values exist. | + | `median()` | Returns the median value, ignoring NaN values. | Returns NaN if any NaN values exist. | + | `count()` | Counts only valid (non-NaN) values. | Counts all values, including NaN. | + | `last()` | Returns the most recent non-NaN value. | Returns the last value, even if it is NaN. | + | `first()` | Returns the oldest non-NaN value. | Returns the first value, even if it is NaN. | + | `sum()` | Computes the total sum, ignoring NaN values. | Returns NaN if any NaN values exist. | + + The following calculations are executed over the provided above arguments: + + **Usage: default (`ignoreNaN = true`)** + + ```javascript + var avgTemp = temperature.mean(); + var tempMax = temperature.max(); + var valueCount = temperature.count(); + ``` + + **Output:** + + ```json + { + "avgTemp": 72.92, + "tempMax": 73.58, + "valueCount": 3 + } + ``` + + **Usage: explicit (`ignoreNaN = false`)** + + ```javascript + var avgTemp = temperature.mean(false); // Returns NaN if any NaN values exist + var tempMax = temperature.max(false); // Returns NaN if any NaN values exist + var valueCount = temperature.count(false); // Counts all values, including NaN + ``` + + **Output:** + + ```json + { + "avgTemp": "NaN", + "tempMax": "NaN", + "valueCount": 4 + } + ``` + + Time series rolling arguments can be **merged** to align timestamps across multiple datasets. + + | Method | Description | Parameters | Returns | + |:-----------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------| + | `merge(other, settings)` | Merges the current rolling argument with another rolling argument by aligning timestamps and filling missing values with the previous available value. |
    • `other` (another rolling argument)
    • `settings` (optional) - configuration object, supports:

      • `ignoreNaN` (boolean, default true) - controls whether NaN values should be ignored.
      • `timeWindow` (object, default {}) - defines a custom time window for filtering merged values.
    | Merged object with timeWindow and aligned values. | + | `mergeAll(others, settings)` | Merges the current rolling argument with multiple rolling arguments by aligning timestamps and filling missing values with the previous available value. |
    • `others` (array of rolling arguments)
    • `settings` (optional) - configuration object, supports:

      • `ignoreNaN` (boolean, default true) - controls whether NaN values should be ignored.
      • `timeWindow` (object, default {}) - defines a custom time window for filtering merged values.
    | Merged object with timeWindow and aligned values.| + + **Example arguments:** + + ```json + { + "humidity": { + "timeWindow": { + "startTs": 1741356332086, + "endTs": 1741357232086 + }, + "values": [{ + "ts": 1741356882759, + "value": 43 + }, { + "ts": 1741356918779, + "value": 46 + }] + }, + "pressure": { + "timeWindow": { + "startTs": 1741356332086, + "endTs": 1741357232086 + }, + "values": [{ + "ts": 1741357047945, + "value": 1023 + }, { + "ts": 1741357056144, + "value": 1026 + }, { + "ts": 1741357147391, + "value": 1025 + }] + }, + "temperature": { + "timeWindow": { + "startTs": 1741356332086, + "endTs": 1741357232086 + }, + "values": [{ + "ts": 1741356874943, + "value": 76 + }, { + "ts": 1741357063689, + "value": 77 + }] + } + } + ``` + + **Usage:** + + ```javascript + var mergedData = temperature.merge(humidity, { ignoreNaN: false }); + ``` + + **Output:** + + ```json + { + "mergedData": { + "timeWindow": { + "startTs": 1741356332086, + "endTs": 1741357232086 + }, + "values": [{ + "ts": 1741356874943, + "values": [76.0, "NaN"] + }, { + "ts": 1741356882759, + "values": [76.0, 43.0] + }, { + "ts": 1741356918779, + "values": [76.0, 46.0] + }, { + "ts": 1741357063689, + "values": [77.0, 46.0] + }] + } + } + ``` + + **Usage:** + + ```javascript + var mergedData = temperature.mergeAll([humidity, pressure], { ignoreNaN: true }); + ``` + + **Output:** + + ```json + { + "mergedData": { + "timeWindow": { + "startTs": 1741356332086, + "endTs": 1741357232086 + }, + "values": [{ + "ts": 1741357047945, + "values": [76.0, 46.0, 1023.0] + }, { + "ts": 1741357056144, + "values": [76.0, 46.0, 1026.0] + }, { + "ts": 1741357063689, + "values": [77.0, 46.0, 1026.0] + }, { + "ts": 1741357147391, + "values": [77.0, 46.0, 1025.0] + }] + } + } + ``` + +##### Function arguments + +* `ctx` - context object that contains all provided arguments, in the representations described above. + +Accessing arguments via `ctx`: + + ```javascript + var altitude = ctx.args.altitude; // single value argument + var temperature = ctx.args.temperature; // time series rolling argument + ``` + +* `arg1, arg2, ...` - user-defined arguments configured in the calculated field setup. + +How they are passed depends on their type: + +* **single value arguments** are passed as raw values **(e.g., 22.5, "ON")**. +* **time series rolling arguments** are passed as objects (containing multiple values). + +##### Example: Air Density Calculation + +This function calculates air density using `altitude` (single value argument) and `temperature` (time series rolling argument). + +```javascript +function calculate(ctx, altitude, temperature) { + var avgTemperature = temperature.mean(); // Get average temperature + var temperatureK = (avgTemperature - 32) * (5 / 9) + 273.15; // Convert Fahrenheit to Kelvin + + // Estimate air pressure based on altitude + var pressure = 101325 * Math.pow((1 - 2.25577e-5 * altitude), 5.25588); + + // Air density formula + var airDensity = pressure / (287.05 * temperatureK); + + return { + "airDensity": airDensity + }; +} +``` + +* `altitude` is a single value, passed as number. +* `temperature` is a rolling argument, retaining its full structure. + +##### Function return format + +The script should return a JSON object formatted according to the [ThingsBoard Telemetry Upload API](${siteBaseUrl}/docs${docPlatformPrefix}/user-guide/telemetry/#time-series-data-upload-api/). +The return value must match one of the supported telemetry upload formats. + +The script must return data in a format compatible with ThingsBoard’s APIs. +The correct return format depends on the calculated field output configuration: + +* if latest telemetry is used for output, the function must return data according to the [Telemetry Upload API](${siteBaseUrl}/docs${docPlatformPrefix}/reference/mqtt-api/#attributes-api/). + + * without timestamps + ```json + { + "airDensity": 1.06, + "someKey": "value" + } + ``` + * with a timestamp: + ```json + { + "ts": 1740644636669, + "values": { + "airDensity": 1.06, + "someKey": "value" + } + } + ``` + +* if attributes are used for output, the function must return data according to the [Attributes API](${siteBaseUrl}/docs${docPlatformPrefix}/user-guide/telemetry/#time-series-data-upload-api/). + + ```json + { + "airDensity": 1.06, + "someKey": "value" + } + ``` diff --git a/ui-ngx/src/assets/help/en_US/rulenode/save_attributes_node_advanced.md b/ui-ngx/src/assets/help/en_US/rulenode/save_attributes_node_advanced.md new file mode 100644 index 0000000000..f0705baa6f --- /dev/null +++ b/ui-ngx/src/assets/help/en_US/rulenode/save_attributes_node_advanced.md @@ -0,0 +1,32 @@ +#### Potential unexpected behavior with mixed processing strategies + +When configuring the processing strategies, certain combinations can lead to unexpected behavior. Consider the following scenarios: + +- **Skipping database storage** + + Choosing to disable attribute persistence introduces the risk of having only partial data available. + For example, if a message is processed solely for real-time notifications via WebSockets and not stored in the database, then attribute queries might not reflect the data visible on the dashboard. + +- **Disabling WebSocket (WS) updates** + + If WS updates are disabled, any changes to the attribute data won’t be pushed to dashboards (or other WS subscriptions). + This means that even if a database is updated, dashboards may not display the updated data until browser page is reloaded. + +- **Skipping calculated field recalculation** + + If attribute data is saved to the database while bypassing calculated field recalculation, the aggregated value may not update to reflect the saved data. + Conversely, if the calculated field is recalculated with new data but the corresponding attribute value is not persisted in the database, the calculated field's value might include data that isn’t stored. + +- **Different deduplication intervals across actions** + + When you configure different deduplication intervals for actions, the same incoming message might be processed differently for each action. + For example, a message might be stored immediately in the Attributes table (if set to *On every message*) while not being present on a dashboard because its deduplication interval hasn’t elapsed. + +- **Deduplication cache clearing** + + The deduplication mechanism uses a cache to track processed messages within each interval. + For performance and system stability reasons, this cache is periodically cleared. + As a result, if a cache entry is removed during the deduplication period, messages from the same originator may be processed more than once within that interval. + This means deduplication should be used as a performance optimization rather than an absolute guarantee of single processing per interval. + +We recommend using deduplication only when the occasional repeated processing is acceptable and won't cause system correctness issue or data inconsistencies. diff --git a/ui-ngx/src/assets/help/en_US/rulenode/save_timeseries_node_advanced.md b/ui-ngx/src/assets/help/en_US/rulenode/save_timeseries_node_advanced.md index 44a037f180..18c0f60cdb 100644 --- a/ui-ngx/src/assets/help/en_US/rulenode/save_timeseries_node_advanced.md +++ b/ui-ngx/src/assets/help/en_US/rulenode/save_timeseries_node_advanced.md @@ -2,23 +2,28 @@ When configuring the processing strategies, certain combinations can lead to unexpected behavior. Consider the following scenarios: +- **Skipping database storage** + + Choosing to disable one or more persistence actions (for instance, skipping database storage for Time series or Latest values while keeping WS updates enabled) introduces the risk of having only partial data available: + - If a message is processed only for real-time notifications (WebSockets) and not stored in the database, historical queries may not match data on the dashboard. + - When processing strategies for Time series and Latest values are out-of-sync, telemetry data may be stored in one table (e.g., Time series) while the same data is absent in the other (e.g., Latest values). + - **Disabling WebSocket (WS) updates** If WS updates are disabled, any changes to the time series data won’t be pushed to dashboards (or other WS subscriptions). This means that even if a database is updated, dashboards may not display the updated data until browser page is reloaded. +- **Skipping calculated field recalculation** + + If telemetry data is saved to the database while bypassing calculated field recalculation, the aggregated value may not update to reflect the latest data. + Conversely, if the calculated field is recalculated with new data but the corresponding telemetry value is not persisted in the database, the calculated field's value might include data that isn’t stored. + - **Different deduplication intervals across actions** When you configure different deduplication intervals for actions, the same incoming message might be processed differently for each action. For example, a message might be stored immediately in the Time series table (if set to *On every message*) while not being stored in the Latest values table because its deduplication interval hasn’t elapsed. Also, if the WebSocket updates are configured with a different interval, dashboards might show updates that do not match what is stored in the database. -- **Skipping database storage** - - Choosing to disable one or more persistence actions (for instance, skipping database storage for Time series or Latest values while keeping WS updates enabled) introduces the risk of having only partial data available: - - If a message is processed only for real-time notifications (WebSockets) and not stored in the database, historical queries may not match data on the dashboard. - - When processing strategies for Time series and Latest values are out-of-sync, telemetry data may be stored in one table (e.g., Time series) while the same data is absent in the other (e.g., Latest values). - - **Deduplication cache clearing** The deduplication mechanism uses a cache to track processed messages within each interval. diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 18801bda2c..85e4daf264 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -65,6 +65,7 @@ "next-with-label": "Next: {{label}}", "read-more": "Read more", "hide": "Hide", + "test": "Test", "done": "Done", "print": "Print", "restore": "Restore", @@ -996,6 +997,7 @@ "failures": "Failures", "entity": "entity", "rule-node": "rule node", + "calculated-field": "calculated field", "hint": { "main": "All node debug messages rate limited with:", "main-limited": "All {{entity}} debug messages will be rate-limited, with a maximum of {{msg}} messages allowed per {{time}}.", @@ -1003,6 +1005,78 @@ "all-messages": "Save all debug events during time limit." } }, + "calculated-fields": { + "expression": "Expression", + "no-found": "No calculated fields found", + "list": "{ count, plural, =1 {One calculated field} other {List of # calculated fields} }", + "selected-fields": "{ count, plural, =1 {1 calculated field} other {# calculated fields} } selected", + "type": { + "simple": "Simple", + "script": "Script" + }, + "arguments": "Arguments", + "decimals-by-default": "Decimals by default", + "debugging": "Calculated field debugging", + "argument-name": "Argument name", + "datasource": "Datasource", + "add-argument": "Add argument", + "test-script-function": "Test script function", + "no-arguments": "No arguments configured", + "argument-settings": "Argument settings", + "argument-current": "Current entity", + "argument-current-tenant": "Current tenant", + "argument-device": "Device", + "argument-asset": "Asset", + "argument-customer": "Customer", + "argument-tenant": "Current tenant", + "argument-type": "Argument type", + "see-debug-events": "See debug events", + "attribute": "Attribute", + "copy-argument-name": "Copy argument name", + "timeseries-key": "Time series key", + "device-name": "Device name", + "latest-telemetry": "Latest telemetry", + "rolling": "Time series rolling", + "attribute-scope": "Attribute scope", + "server-attributes": "Server attributes", + "client-attributes": "Client attributes", + "shared-attributes": "Shared attributes", + "attribute-key": "Attribute key", + "default-value": "Default value", + "limit": "Max values", + "time-window": "Time window", + "customer-name": "Customer name", + "asset-name": "Asset name", + "timeseries": "Time series", + "output": "Output", + "create": "Create new calculated field", + "file": "Calculated field file", + "invalid-file-error": "Invalid file format. Please make sure the file is a valid JSON file.", + "import": "Import calculated field", + "export": "Export calculated field", + "export-failed-error": "Unable to export calculated field: {{error}}", + "output-type": "Output type", + "delete-title": "Are you sure you want to delete the calculated field '{{title}}'?", + "delete-text": "Be careful, after the confirmation the calculated field and all related data will become unrecoverable.", + "delete-multiple-title": "Are you sure you want to delete { count, plural, =1 {1 calculated field} other {# calculated fields} }?", + "delete-multiple-text": "Be careful, after the confirmation all selected calculated fields will be removed and all related data will become unrecoverable.", + "hint": { + "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with time series rolling type.", + "arguments-empty": "Arguments should not be empty.", + "expression-required": "Expression is required.", + "expression-invalid": "Expression is invalid", + "expression-max-length": "Expression length should be less than 255 characters.", + "argument-name-required": "Argument name is required.", + "argument-name-pattern": "Argument name is invalid.", + "argument-name-duplicate": "Argument with such name already exists.", + "argument-name-max-length": "Argument name should be less than 256 characters.", + "argument-name-ctx": "Argument name 'ctx' is reserved and cannot be used.", + "argument-type-required": "Argument type is required.", + "max-args": "Maximum number of arguments reached.", + "decimals-range": "Decimals by default should be a number between 0 and 15.", + "expression": "Default expression demonstrates how to transform a temperature from Fahrenheit to Celsius." + } + }, "confirm-on-exit": { "message": "You have unsaved changes. Are you sure you want to leave this page?", "html-message": "You have unsaved changes.
    Are you sure you want to leave this page?", @@ -1027,8 +1101,13 @@ "city-max-length": "Specified city should be less than 256" }, "common": { + "name": "Name", + "type": "Type", + "general": "General", "username": "Username", "password": "Password", + "data": "Data", + "timestamp": "Timestamp", "enter-username": "Enter username", "enter-password": "Enter password", "enter-search": "Enter search", @@ -1038,8 +1117,27 @@ "proceed": "Proceed", "open-details-page": "Open details page", "not-found": "Not found", + "value": "Value", "documentation": "Documentation", - "time-left": "{{time}} left" + "time-left": "{{time}} left", + "output": "Output", + "test-function": "Test function", + "test-with-this-message": "{{test}} with this message", + "suffix": { + "s": "s", + "ms": "ms" + }, + "hint": { + "name-required": "Name is required.", + "name-pattern": "Name is invalid.", + "name-max-length": "Name should be less than 256 characters.", + "title-required": "Title is required.", + "title-pattern": "Title is invalid.", + "title-max-length": "Title should be less than 256 characters.", + "key-required": "Key is required.", + "key-pattern": "Key is invalid.", + "key-max-length": "Key should be less than 256 characters." + } }, "content-type": { "json": "Json", @@ -2427,6 +2525,8 @@ "type-current-tenant": "Current Tenant", "type-current-user": "Current User", "type-current-user-owner": "Current User Owner", + "type-calculated-field": "Calculated field", + "type-calculated-fields": "Calculated fields", "type-widgets-bundle": "Widgets bundle", "type-widgets-bundles": "Widgets bundles", "list-of-widgets-bundles": "{ count, plural, =1 {One widgets bundle} other {List of # widget bundles} }", @@ -2627,6 +2727,9 @@ "type-stats": "Statistics", "type-debug-rule-node": "Debug", "type-debug-rule-chain": "Debug", + "type-debug-calculated-field": "Debug", + "arguments": "Arguments", + "result": "Result", "no-events-prompt": "No events found", "error": "Error", "alarm": "Alarm", @@ -4265,6 +4368,7 @@ "delete-javascript-resources-action-title": "Delete JavaScript { count, plural, =1 {1 resource} other {# resources} }", "delete-javascript-resources-text": "Please note that the selected JavaScript resources, even if they are used in JavaScript functions, will be deleted.", "delete-javascript-resources-title": "Are you sure you want to delete JavaScript { count, plural, =1 {1 resource} other {# resources} }?", + "delete-javascript-resource-in-use-text": "If you still want to delete the JavaScript resource, click the Delete anyway button.", "download": "Download JavaScript resource", "upload-from-file": "Upload JavaScript from file", "resource-file": "JavaScript resource file", @@ -4273,6 +4377,10 @@ "javascript-library": "JavaScript library", "javascript-type": "JavaScript type", "javascript-resource-details": "JavaScript resource details", + "javascript-resource-is-in-use": "JavaScript resource is used by other entities", + "javascript-resources-are-in-use": "JavaScript resources are used by other entities", + "javascript-resource-is-in-use-text": "The JavaScript resource '{{title}}' was not deleted because it is used by the following entities:", + "javascript-resources-are-in-use-text": "Not all JavaScript resources have been deleted because they are used by other entities.
    You can view referenced entities by clicking the References button in the corresponding resource row.
    If you still want to delete these JavaScript resources, select them in the table below and click the Delete selected button.", "search": "Search JavaScript resources", "selected-javascript-resources": "{ count, plural, =1 {1 JavaScript resource} other {# JavaScript resources} } selected", "no-javascript-resource-text": "No JavaScript resources found", @@ -5153,7 +5261,25 @@ }, "time-series": "Time series", "latest": "Latest values", - "web-sockets": "WebSockets" + "web-sockets": "WebSockets", + "calculated-fields": "Calculated fields" + }, + "save-attribute": { + "processing-settings": "Processing settings", + "processing-settings-hint": "Define how incoming messages are processed. In Basic mode, select a preconfigured processing strategy or enable only WebSocket updates. Advanced mode allows you to select individual processing strategies for each action.", + "advanced-settings-hint": "Be cautious when configuring processing strategies. Certain combinations can lead to unexpected behavior.", + "strategy": "Strategy", + "deduplication-interval": "Deduplication interval", + "deduplication-interval-required": "Deduplication interval is required", + "deduplication-interval-min-max-range": "Deduplication interval should be at least 1 second and at most 1 day", + "scope": "Scope", + "strategy-type": { + "every-message": "On every message", + "skip": "Skip", + "deduplicate": "Deduplicate", + "web-sockets-only": "WebSockets only" + }, + "attributes": "Attributes" }, "key-val": { "key": "Key", @@ -5376,6 +5502,7 @@ "entities": "Entities", "rule-engine": "Rule Engine", "time-to-live": "Time-to-live", + "calculated-fields": "Calculated fields", "alarms-and-notifications": "Alarms and notifications", "ota-files-in-bytes": "Files", "ws-title": "WS", @@ -5428,6 +5555,21 @@ "tenant-entity-import-rate-limit": "Entity version load", "tenant-notification-request-rate-limit": "Notification requests", "tenant-notification-requests-per-rule-rate-limit": "Notification requests per notification rule", + "max-calculated-fields": "Calculated fields per entity maximum number", + "max-calculated-fields-range": "Calculated fields per entity maximum number can't be negative", + "max-calculated-fields-required": "Calculated fields per entity maximum number is required", + "max-data-points-per-rolling-arg": "Maximum data points number in rolling arguments", + "max-data-points-per-rolling-arg-range": "Maximum data points number in rolling arguments can't be negative", + "max-data-points-per-rolling-arg-required": "Maximum data points number in rolling arguments is required", + "max-arguments-per-cf": "Arguments per calculated field maximum number", + "max-arguments-per-cf-range": "Arguments per calculated field maximum number can't be negative", + "max-arguments-per-cf-required": "Arguments per calculated field maximum number is required", + "max-state-size": "State maximum size in KB", + "max-state-size-range": "State maximum size in KB can't be negative", + "max-state-size-required": "State maximum size in KB is required", + "max-value-argument-size": "Single value argument maximum size in KB", + "max-value-argument-size-range": "Single value argument maximum size in KB can't be negative", + "max-value-argument-size-required": "Single value argument maximum size in KB is required", "max-transport-messages": "Transport messages maximum number", "max-transport-messages-required": "Transport messages maximum number is required.", "max-transport-messages-range": "Transport messages maximum number can't be negative", @@ -5499,6 +5641,8 @@ "advanced-settings": "Advanced settings", "edit-limit": "Edit limit", "but-less-than": "but less than", + "calculated-field-debug-event-rate-limit": "Calculated field debug events", + "edit-calculated-field-debug-event-rate-limit": "Edit calculated field debug events rate limits", "edit-transport-tenant-msg-title": "Edit transport tenant messages rate limits", "edit-transport-tenant-telemetry-msg-title": "Edit transport tenant telemetry messages rate limits", "edit-transport-tenant-telemetry-data-points-title": "Edit transport tenant telemetry data points rate limits", @@ -6197,6 +6341,7 @@ "export-relations": "Export relations", "export-attributes": "Export attributes", "export-credentials": "Export credentials", + "export-calculated-fields": "Export calculated fields", "entity-versions": "Entity versions", "versions": "Versions", "created-time": "Created time", @@ -6213,6 +6358,7 @@ "load-relations": "Load relations", "load-attributes": "Load attributes", "load-credentials": "Load credentials", + "load-calculated-fields": "Load calculated fields", "compare-with-current": "Compare with current", "diff-entity-with-version": "Diff with entity version '{{versionName}}'", "previous-difference": "Previous Difference", diff --git a/ui-ngx/src/form.scss b/ui-ngx/src/form.scss index e42fac19c3..8f3c7f0cd0 100644 --- a/ui-ngx/src/form.scss +++ b/ui-ngx/src/form.scss @@ -163,6 +163,13 @@ .tb-form-panel-title { font-weight: 500; font-size: 16px; + + &.tb-required::after { + font-size: 13px; + color: rgba(0, 0, 0, .54); + vertical-align: top; + content: " *"; + } } .tb-form-panel-hint { font-size: 12px;