From e5003df778701a7191a577fb784d97cfc0c47eb4 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Mon, 30 Dec 2024 20:30:44 +0200 Subject: [PATCH 01/30] [WIP] save time series strategies: draft implementation for latest using Bloom filter --- .../DoNotSavePersistenceStrategy.java | 34 ++++++++++++ .../engine/telemetry/PersistenceStrategy.java | 35 ++++++++++++ .../SaveEveryMessagePersistenceStrategy.java | 34 ++++++++++++ ...aveFirstInIntervalPersistenceStrategy.java | 54 +++++++++++++++++++ .../engine/telemetry/TbMsgTimeseriesNode.java | 52 +++++++++++++++--- .../TbMsgTimeseriesNodeConfiguration.java | 12 ++++- .../telemetry/TbMsgTimeseriesNodeTest.java | 4 +- 7 files changed, 213 insertions(+), 12 deletions(-) create mode 100644 rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/DoNotSavePersistenceStrategy.java create mode 100644 rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/PersistenceStrategy.java create mode 100644 rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/SaveEveryMessagePersistenceStrategy.java create mode 100644 rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/SaveFirstInIntervalPersistenceStrategy.java diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/DoNotSavePersistenceStrategy.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/DoNotSavePersistenceStrategy.java new file mode 100644 index 0000000000..5652a34cd1 --- /dev/null +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/DoNotSavePersistenceStrategy.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.telemetry; + +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.UUID; + +public final class DoNotSavePersistenceStrategy implements PersistenceStrategy { + + public static final DoNotSavePersistenceStrategy INSTANCE = new DoNotSavePersistenceStrategy(); + + private DoNotSavePersistenceStrategy() { + } + + @Override + public boolean shouldPersist(UUID originatorUuid, TsKvEntry timeseriesEntry) { + return false; + } + +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/PersistenceStrategy.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/PersistenceStrategy.java new file mode 100644 index 0000000000..2aa92bd57c --- /dev/null +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/PersistenceStrategy.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.telemetry; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.UUID; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = SaveEveryMessagePersistenceStrategy.class, name = "SAVE_EVERY_MESSAGE"), + @JsonSubTypes.Type(value = SaveFirstInIntervalPersistenceStrategy.class, name = "SAVE_FIRST_IN_INTERVAL"), + @JsonSubTypes.Type(value = DoNotSavePersistenceStrategy.class, name = "DO_NOT_SAVE") +}) +public sealed interface PersistenceStrategy permits DoNotSavePersistenceStrategy, SaveEveryMessagePersistenceStrategy, SaveFirstInIntervalPersistenceStrategy { + + // TODO: maybe this should accept generic key? + boolean shouldPersist(UUID originatorUuid, TsKvEntry timeseriesEntry); + +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/SaveEveryMessagePersistenceStrategy.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/SaveEveryMessagePersistenceStrategy.java new file mode 100644 index 0000000000..7ca564b0bc --- /dev/null +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/SaveEveryMessagePersistenceStrategy.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.telemetry; + +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.UUID; + +public final class SaveEveryMessagePersistenceStrategy implements PersistenceStrategy { + + public static final SaveEveryMessagePersistenceStrategy INSTANCE = new SaveEveryMessagePersistenceStrategy(); + + private SaveEveryMessagePersistenceStrategy() { + } + + @Override + public boolean shouldPersist(UUID originatorUuid, TsKvEntry timeseriesEntry) { + return true; + } + +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/SaveFirstInIntervalPersistenceStrategy.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/SaveFirstInIntervalPersistenceStrategy.java new file mode 100644 index 0000000000..4677058e6a --- /dev/null +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/SaveFirstInIntervalPersistenceStrategy.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.telemetry; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.hash.BloomFilter; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +@SuppressWarnings("UnstableApiUsage") +public final class SaveFirstInIntervalPersistenceStrategy implements PersistenceStrategy { + + private final long intervalDurationMillis; + private final BloomFilter filter; + + @JsonCreator + public SaveFirstInIntervalPersistenceStrategy(@JsonProperty("intervalDurationMillis") long intervalDurationMillis) { + this.intervalDurationMillis = intervalDurationMillis; + // TODO: implement funnel as an enum + filter = BloomFilter.create((key, sink) -> + sink.putLong(key.intervalNumber()) + .putLong(key.originatorUuid().getMostSignificantBits()) + .putLong(key.originatorUuid().getLeastSignificantBits()) + .putString(key.timeseriesKey(), StandardCharsets.UTF_8), 1_000_000); + } + + // TODO: this should not be hardcoded here but should be defined by clients + // should be generified (what to do with funnel then?) + private record Key(long intervalNumber, UUID originatorUuid, String timeseriesKey) { + } + + @Override + public boolean shouldPersist(UUID originatorUuid, TsKvEntry timeseriesEntry) { + long intervalNumber = timeseriesEntry.getTs() / intervalDurationMillis; + return filter.put(new Key(intervalNumber, originatorUuid, timeseriesEntry.getKey())); + } + +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 27f45feb47..28f9c88e34 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -15,8 +15,12 @@ */ package org.thingsboard.rule.engine.telemetry; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; import com.google.gson.JsonParser; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.rule.engine.api.RuleNode; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNode; @@ -27,6 +31,9 @@ import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -59,7 +66,7 @@ import static org.thingsboard.server.common.data.msg.TbMsgType.POST_TELEMETRY_RE "The DB layer has certain optimizations to ignore the updates of the \"attributes\" and \"latest values\" tables if the new record has a timestamp that is older than the previous record. " + "So, to make sure that all the messages will be processed correctly, one should enable this parameter for sequential message processing scenarios.", uiResources = {"static/rulenode/rulenode-core-config.js"}, - configDirective = "tbActionNodeTimeseriesConfig", + // configDirective = "tbActionNodeTimeseriesConfig", icon = "file_upload" ) public class TbMsgTimeseriesNode implements TbNode { @@ -68,10 +75,13 @@ public class TbMsgTimeseriesNode implements TbNode { private TbContext ctx; private long tenantProfileDefaultStorageTtl; + private PersistenceStrategy latestPersistenceStrategy; + @Override public void init(TbContext ctx, TbNodeConfiguration configuration) throws TbNodeException { this.config = TbNodeUtils.convert(configuration, TbMsgTimeseriesNodeConfiguration.class); this.ctx = ctx; + latestPersistenceStrategy = config.getPersistenceConfig().latest(); ctx.addTenantProfileListener(this::onTenantProfileUpdate); onTenantProfileUpdate(ctx.getTenantProfile()); } @@ -94,10 +104,18 @@ public class TbMsgTimeseriesNode implements TbNode { ctx.tellFailure(msg, new IllegalArgumentException("Msg body is empty: " + src)); return; } - List tsKvEntryList = new ArrayList<>(); + List withLatest = new ArrayList<>(); + List withoutLatest = new ArrayList<>(); for (Map.Entry> tsKvEntry : tsKvMap.entrySet()) { for (KvEntry kvEntry : tsKvEntry.getValue()) { - tsKvEntryList.add(new BasicTsKvEntry(tsKvEntry.getKey(), kvEntry)); + TsKvEntry entry = new BasicTsKvEntry(tsKvEntry.getKey(), kvEntry); + if (latestPersistenceStrategy.shouldPersist(msg.getOriginator().getId(), entry)) { + log.info("Persisting entry: {}", entry); + withLatest.add(entry); + } else { + log.info("Skipping entry: {}", entry); + withoutLatest.add(entry); + } } } String ttlValue = msg.getMetaData().getValue("TTL"); @@ -105,15 +123,33 @@ public class TbMsgTimeseriesNode implements TbNode { if (ttl == 0L) { ttl = tenantProfileDefaultStorageTtl; } - ctx.getTelemetryService().saveTimeseries(TimeseriesSaveRequest.builder() + + SettableFuture withLatestSavedFuture = SettableFuture.create(); + TimeseriesSaveRequest saveWithLatestRequest = TimeseriesSaveRequest.builder() + .tenantId(ctx.getTenantId()) + .customerId(msg.getCustomerId()) + .entityId(msg.getOriginator()) + .entries(withLatest) + .ttl(ttl) + .saveLatest(true) + .future(withLatestSavedFuture) + .build(); + ctx.getTelemetryService().saveTimeseries(saveWithLatestRequest); + + SettableFuture withoutLatestSavedFuture = SettableFuture.create(); + TimeseriesSaveRequest saveWithoutLatestRequest = TimeseriesSaveRequest.builder() .tenantId(ctx.getTenantId()) .customerId(msg.getCustomerId()) .entityId(msg.getOriginator()) - .entries(tsKvEntryList) + .entries(withoutLatest) .ttl(ttl) - .saveLatest(!config.isSkipLatestPersistence()) - .callback(new TelemetryNodeCallback(ctx, msg)) - .build()); + .saveLatest(false) + .future(withoutLatestSavedFuture) + .build(); + ctx.getTelemetryService().saveTimeseries(saveWithoutLatestRequest); + + ListenableFuture> bothSavedFuture = Futures.allAsList(withLatestSavedFuture, withoutLatestSavedFuture); + DonAsynchron.withCallback(bothSavedFuture, success -> ctx.tellSuccess(msg), failure -> ctx.tellFailure(msg, failure)); } public static long computeTs(TbMsg msg, boolean ignoreMetadataTs) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java index 1c33778a6b..adaebb0967 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java @@ -15,6 +15,7 @@ */ package org.thingsboard.rule.engine.telemetry; +import lombok.Builder; import lombok.Data; import org.thingsboard.rule.engine.api.NodeConfiguration; @@ -22,15 +23,22 @@ import org.thingsboard.rule.engine.api.NodeConfiguration; public class TbMsgTimeseriesNodeConfiguration implements NodeConfiguration { private long defaultTTL; - private boolean skipLatestPersistence; private boolean useServerTs; + private PersistenceConfig persistenceConfig; @Override public TbMsgTimeseriesNodeConfiguration defaultConfiguration() { TbMsgTimeseriesNodeConfiguration configuration = new TbMsgTimeseriesNodeConfiguration(); configuration.setDefaultTTL(0L); - configuration.setSkipLatestPersistence(false); configuration.setUseServerTs(false); + configuration.setPersistenceConfig(PersistenceConfig.builder() + .latest(SaveEveryMessagePersistenceStrategy.INSTANCE) + .build()); return configuration; } + + @Builder + record PersistenceConfig(PersistenceStrategy latest) { + } + } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java index 2cba4b8fb3..74240d3f97 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java @@ -88,7 +88,7 @@ public class TbMsgTimeseriesNodeTest { @Test public void verifyDefaultConfig() { assertThat(config.getDefaultTTL()).isEqualTo(0L); - assertThat(config.isSkipLatestPersistence()).isFalse(); + // assertThat(config.isSkipLatestPersistence()).isFalse(); assertThat(config.isUseServerTs()).isFalse(); } @@ -162,7 +162,7 @@ public class TbMsgTimeseriesNodeTest { public void givenSkipLatestPersistenceIsTrueAndTtlFromConfig_whenOnMsg_thenSaveTimeseriesUsingTtlFromConfig() throws TbNodeException { long ttlFromConfig = 5L; config.setDefaultTTL(ttlFromConfig); - config.setSkipLatestPersistence(true); + // config.setSkipLatestPersistence(true); init(); String data = """ From 345e4239732c2c801878445cd65ee369b61160cc Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Sun, 12 Jan 2025 14:27:46 +0200 Subject: [PATCH 02/30] Save time series strategies: update time series node config in rule chain JSONs; add null-check for persistence settings --- .../rule_chains/edge_root_rule_chain.json | 8 ++++++-- .../device_profile/rule_chain_template.json | 8 ++++++-- .../tenant/rule_chains/root_rule_chain.json | 6 +++++- .../src/main/resources/root_rule_chain.json | 20 ++++++++++++++----- .../engine/telemetry/TbMsgTimeseriesNode.java | 3 +++ 5 files changed, 35 insertions(+), 10 deletions(-) diff --git a/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json b/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json index 9adeb4f49e..e7b4b93e98 100644 --- a/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json +++ b/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json @@ -34,7 +34,11 @@ "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries", "configuration": { - "defaultTTL": 0 + "defaultTTL": 0, + "useServerTs": false, + "persistenceSettings": { + "type": "ON_EVERY_MESSAGE" + } }, "externalId": null }, @@ -185,4 +189,4 @@ ], "ruleChainConnections": null } -} \ No newline at end of file +} diff --git a/application/src/main/data/json/tenant/device_profile/rule_chain_template.json b/application/src/main/data/json/tenant/device_profile/rule_chain_template.json index bce88d62b0..37cc226a6e 100644 --- a/application/src/main/data/json/tenant/device_profile/rule_chain_template.json +++ b/application/src/main/data/json/tenant/device_profile/rule_chain_template.json @@ -20,7 +20,11 @@ "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries", "configuration": { - "defaultTTL": 0 + "defaultTTL": 0, + "useServerTs": false, + "persistenceSettings": { + "type": "ON_EVERY_MESSAGE" + } } }, { @@ -134,4 +138,4 @@ ], "ruleChainConnections": null } -} \ No newline at end of file +} diff --git a/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json b/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json index ee38849c1b..7893a64fd2 100644 --- a/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json +++ b/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json @@ -19,7 +19,11 @@ "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries", "configuration": { - "defaultTTL": 0 + "defaultTTL": 0, + "useServerTs": false, + "persistenceSettings": { + "type": "ON_EVERY_MESSAGE" + } } }, { diff --git a/monitoring/src/main/resources/root_rule_chain.json b/monitoring/src/main/resources/root_rule_chain.json index ff44ebfe79..a48fd6bf35 100644 --- a/monitoring/src/main/resources/root_rule_chain.json +++ b/monitoring/src/main/resources/root_rule_chain.json @@ -23,7 +23,11 @@ "singletonMode": false, "configurationVersion": 0, "configuration": { - "defaultTTL": 0 + "defaultTTL": 0, + "useServerTs": false, + "persistenceSettings": { + "type": "ON_EVERY_MESSAGE" + } }, "externalId": null }, @@ -275,7 +279,11 @@ "singletonMode": false, "configurationVersion": 0, "configuration": { - "defaultTTL": 0 + "defaultTTL": 0, + "useServerTs": false, + "persistenceSettings": { + "type": "ON_EVERY_MESSAGE" + } }, "externalId": null }, @@ -310,8 +318,10 @@ "configurationVersion": 0, "configuration": { "defaultTTL": 180, - "skipLatestPersistence": null, - "useServerTs": null + "useServerTs": false, + "persistenceSettings": { + "type": "ON_EVERY_MESSAGE" + } }, "externalId": null } @@ -415,4 +425,4 @@ ], "ruleChainConnections": null } -} \ No newline at end of file +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index b3eea21ece..1a014c5a43 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -83,6 +83,9 @@ public class TbMsgTimeseriesNode implements TbNode { ctx.addTenantProfileListener(this::onTenantProfileUpdate); onTenantProfileUpdate(ctx.getTenantProfile()); persistenceSettings = config.getPersistenceSettings(); + if (persistenceSettings == null) { + throw new TbNodeException("Persistence settings cannot be null!", true); + } } void onTenantProfileUpdate(TenantProfile tenantProfile) { From f8cfd158e279107e784014790accd2f689eb0b5a Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Sun, 12 Jan 2025 14:35:21 +0200 Subject: [PATCH 03/30] Save time series strategies: update one more time series node config in rule chain JSON --- .../src/test/resources/MqttRuleNodeTestMetadata.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/msa/black-box-tests/src/test/resources/MqttRuleNodeTestMetadata.json b/msa/black-box-tests/src/test/resources/MqttRuleNodeTestMetadata.json index c2bb52514e..adcb618cbe 100644 --- a/msa/black-box-tests/src/test/resources/MqttRuleNodeTestMetadata.json +++ b/msa/black-box-tests/src/test/resources/MqttRuleNodeTestMetadata.json @@ -39,8 +39,10 @@ "configurationVersion": 0, "configuration": { "defaultTTL": 0, - "skipLatestPersistence": false, - "useServerTs": false + "useServerTs": false, + "persistenceSettings": { + "type": "ON_EVERY_MESSAGE" + } }, "externalId": null }, From 698a0c19ecc61d654f61d0a52ea9ec7cdf68314a Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Mon, 13 Jan 2025 12:16:12 +0200 Subject: [PATCH 04/30] Save time series strategies: add SQL upgrade script --- .../main/data/upgrade/basic/schema_update.sql | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index 64bbbaca05..0ff57119f4 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -209,3 +209,74 @@ $$; UPDATE resource SET resource_sub_type = 'EXTENSION' WHERE resource_type = 'JS_MODULE' AND resource_sub_type IS NULL; -- UPDATE RESOURCE JS_MODULE SUB TYPE END + +-- UPDATE SAVE TIME SERIES NODES START + +DO $$ + BEGIN + -- Check if the rule_node table exists + IF EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_name = 'rule_node' + ) THEN + + -- CREATE JSON validation function + CREATE OR REPLACE FUNCTION is_valid_jsonb(input text) + RETURNS boolean + LANGUAGE plpgsql + AS $func$ + DECLARE + dummy JSONB; + BEGIN + dummy := input::jsonb; + RETURN true; + EXCEPTION + WHEN others THEN + RETURN false; + END; + $func$; + + UPDATE rule_node + SET configuration = CASE + -- Case 1: If configuration is NULL, invalid JSON, or not a JSON object - set default configuration + WHEN configuration IS NULL + OR NOT is_valid_jsonb(configuration) + OR jsonb_typeof(configuration::jsonb) <> 'object' + THEN jsonb_build_object( + 'defaultTTL', 0, + 'useServerTs', false, + 'persistenceSettings', jsonb_build_object('type', 'ON_EVERY_MESSAGE') + ) + -- Case 2: If a valid JSON object with persistenceSettings (rule node was already upgraded) - leave unchanged + WHEN configuration::jsonb ? 'persistenceSettings' + THEN configuration::jsonb + -- Case 3: If a valid JSON object without persistenceSettings and skipLatestPersistence = 'true' (string 'true' or boolean true) - set latest to SKIP + WHEN configuration::jsonb ->> 'skipLatestPersistence' = 'true' + THEN (configuration::jsonb - 'skipLatestPersistence') + || jsonb_build_object( + 'persistenceSettings', jsonb_build_object( + 'type', 'ADVANCED', + 'timeseries', jsonb_build_object('type', 'ON_EVERY_MESSAGE'), + 'latest', jsonb_build_object('type', 'SKIP'), + 'webSockets', jsonb_build_object('type', 'ON_EVERY_MESSAGE') + ) + ) + -- Case 4: If a valid JSON object without persistenceSettings and skipLatestPersistence not 'true' (everything else) - set all to ON_EVERY_MESSAGE + ELSE (configuration::jsonb - 'skipLatestPersistence') + || jsonb_build_object( + 'persistenceSettings', jsonb_build_object( + 'type', 'ON_EVERY_MESSAGE' + ) + ) + END::text + WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode'; + + -- Drop the helper function + DROP FUNCTION is_valid_jsonb(text); + + END IF; + END; +$$; + +-- UPDATE SAVE TIME SERIES NODES END From 631d314fcebe4f63c36597ead46549596bc1d2a4 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Mon, 13 Jan 2025 15:54:11 +0200 Subject: [PATCH 05/30] Save time series strategies: add Java upgrade script --- .../main/data/upgrade/basic/schema_update.sql | 5 +- .../engine/telemetry/TbMsgTimeseriesNode.java | 36 +++++++- .../telemetry/TbMsgTimeseriesNodeTest.java | 86 ++++++++++++++++++- 3 files changed, 121 insertions(+), 6 deletions(-) diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index 0ff57119f4..6aaf8c3dbd 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -269,8 +269,9 @@ DO $$ 'type', 'ON_EVERY_MESSAGE' ) ) - END::text - WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode'; + END::text, + configuration_version = 1 + WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode' AND configuration_version = 0; -- Drop the helper function DROP FUNCTION is_valid_jsonb(text); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 1a014c5a43..db2421e307 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -15,8 +15,11 @@ */ package org.thingsboard.rule.engine.telemetry; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.gson.JsonParser; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.RuleNode; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNode; @@ -24,6 +27,7 @@ import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.rule.engine.telemetry.strategy.PersistenceStrategy; import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TenantProfile; @@ -32,6 +36,7 @@ import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.msg.TbMsg; import java.util.ArrayList; @@ -66,7 +71,8 @@ import static org.thingsboard.server.common.data.msg.TbMsgType.POST_TELEMETRY_RE "So, to make sure that all the messages will be processed correctly, one should enable this parameter for sequential message processing scenarios.", uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = "tbActionNodeTimeseriesConfig", - icon = "file_upload" + icon = "file_upload", + version = 1 ) public class TbMsgTimeseriesNode implements TbNode { @@ -182,4 +188,32 @@ public class TbMsgTimeseriesNode implements TbNode { ctx.removeListeners(); } + @Override + public TbPair upgrade(int fromVersion, JsonNode oldConfiguration) throws TbNodeException { + boolean hasChanges = false; + switch (fromVersion) { + case 0: + if (oldConfiguration.has("persistenceSettings")) { + break; + } + hasChanges = true; + JsonNode skipLatestPersistence = oldConfiguration.get("skipLatestPersistence"); + if (skipLatestPersistence != null && "true".equals(skipLatestPersistence.asText())) { + var skipLatestPersistenceSettings = new Advanced( + PersistenceStrategy.onEveryMessage(), + PersistenceStrategy.skip(), + PersistenceStrategy.onEveryMessage() + ); + ((ObjectNode) oldConfiguration).set("persistenceSettings", JacksonUtil.valueToTree(skipLatestPersistenceSettings)); + } else { + ((ObjectNode) oldConfiguration).set("persistenceSettings", JacksonUtil.valueToTree(new OnEveryMessage())); + } + ((ObjectNode) oldConfiguration).remove("skipLatestPersistence"); + break; + default: + break; + } + return new TbPair<>(hasChanges, oldConfiguration); + } + } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java index ba8c614944..71d060aab2 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java @@ -25,11 +25,12 @@ import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; import org.mockito.ArgumentCaptor; import org.mockito.Mock; -import org.mockito.ThrowingConsumer; import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.AbstractRuleNodeUpgradeTest; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; @@ -60,12 +61,13 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.assertArg; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) -public class TbMsgTimeseriesNodeTest { +public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("c8f34868-603a-4433-876a-7d356e5cf377")); private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("e5095e9a-04f4-44c9-b443-1cf1b97d3384")); @@ -82,7 +84,7 @@ public class TbMsgTimeseriesNodeTest { @BeforeEach public void setUp() throws TbNodeException { - node = new TbMsgTimeseriesNode(); + node = spy(new TbMsgTimeseriesNode()); config = new TbMsgTimeseriesNodeConfiguration().defaultConfiguration(); } @@ -291,4 +293,82 @@ public class TbMsgTimeseriesNodeTest { return expectedList; } + @Override + protected TbNode getTestNode() { + return node; + } + + private static Stream givenFromVersionAndConfig_whenUpgrade_thenVerifyHasChangesAndConfig() { + return Stream.of( + Arguments.of(0, """ + { + "defaultTTL": 0, + "useServerTs": false, + "skipLatestPersistence": false + }""", + true, + """ + { + "defaultTTL": 0, + "useServerTs": false, + "persistenceSettings": { + "type": "ON_EVERY_MESSAGE" + } + }"""), + Arguments.of(0, """ + { + "defaultTTL": 0, + "useServerTs": false + }""", + true, + """ + { + "defaultTTL": 0, + "useServerTs": false, + "persistenceSettings": { + "type": "ON_EVERY_MESSAGE" + } + }"""), + Arguments.of(0, """ + { + "defaultTTL": 0, + "useServerTs": false, + "skipLatestPersistence": null + }""", + true, + """ + { + "defaultTTL": 0, + "useServerTs": false, + "persistenceSettings": { + "type": "ON_EVERY_MESSAGE" + } + }"""), + Arguments.of(0, """ + { + "defaultTTL": 0, + "useServerTs": false, + "skipLatestPersistence": true + }""", + true, + """ + { + "defaultTTL": 0, + "useServerTs": false, + "persistenceSettings": { + "type": "ADVANCED", + "timeseries": { + "type": "ON_EVERY_MESSAGE" + }, + "latest": { + "type": "SKIP" + }, + "webSockets": { + "type": "ON_EVERY_MESSAGE" + } + } + }""") + ); + } + } From d1acba40a29391d59df1b5e96359664be1b503ea Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Tue, 14 Jan 2025 15:20:16 +0200 Subject: [PATCH 06/30] Save time series strategies: add tests for changes in telemetry service --- .../DefaultTbEntityViewService.java | 1 + .../DefaultTelemetrySubscriptionService.java | 7 +- ...faultTelemetrySubscriptionServiceTest.java | 387 ++++++++++++++++++ .../engine/api/TimeseriesSaveRequestTest.java | 33 ++ 4 files changed, 425 insertions(+), 3 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java create mode 100644 rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java index 8e0a901eed..ee807da750 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java @@ -350,6 +350,7 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen .entries(latestValues) .saveTimeseries(false) .saveLatest(true) + .sendWsUpdate(true) .callback(new FutureCallback() { @Override public void onSuccess(@Nullable Void tmp) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index d7ff5fa5b7..dbd7967989 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -149,8 +149,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (request.isSendWsUpdate()) { addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); } - if (request.isSaveTimeseries() && request.isSaveLatest()) { - addEntityViewCallback(tenantId, entityId, request.getEntries()); + if (request.isSaveLatest()) { + copyLatestToEntityViews(tenantId, entityId, request.getEntries()); } return saveFuture; } @@ -205,7 +205,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } } - private void addEntityViewCallback(TenantId tenantId, EntityId entityId, List ts) { + private void copyLatestToEntityViews(TenantId tenantId, EntityId entityId, List ts) { if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { Futures.addCallback(this.tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId), new FutureCallback<>() { @@ -238,6 +238,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer .entries(entityViewLatest) .saveTimeseries(false) .saveLatest(true) + .sendWsUpdate(true) .callback(new FutureCallback<>() { @Override public void onSuccess(@Nullable Void tmp) {} diff --git a/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java new file mode 100644 index 0000000000..8162ef4555 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java @@ -0,0 +1,387 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.telemetry; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.SettableFuture; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.ApiUsageRecordKey; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.ApiUsageStateValue; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.objects.AttributesEntityView; +import org.thingsboard.server.common.data.objects.TelemetryEntityView; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.stats.TbApiUsageReportClient; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; +import org.thingsboard.server.service.subscription.SubscriptionManagerService; + +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.stream.LongStream; +import java.util.stream.Stream; + +import static com.google.common.util.concurrent.Futures.immediateFuture; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.then; +import static org.mockito.Mockito.lenient; + +@ExtendWith(MockitoExtension.class) +class DefaultTelemetrySubscriptionServiceTest { + + final TenantId tenantId = TenantId.fromUUID(UUID.fromString("a00ec470-c6b4-11ef-8c88-63b5533fb5bc")); + final CustomerId customerId = new CustomerId(UUID.fromString("7bdc9750-c775-11ef-8e03-ff69ed8da327")); + final EntityId entityId = DeviceId.fromString("cc51e450-53e1-11ee-883e-e56b48fd2088"); + + final long sampleTtl = 10_000L; + + final List sampleTelemetry = List.of( + new BasicTsKvEntry(100L, new DoubleDataEntry("temperature", 65.2)), + new BasicTsKvEntry(100L, new DoubleDataEntry("humidity", 33.1)) + ); + + ApiUsageState apiUsageState; + + final TopicPartitionInfo tpi = TopicPartitionInfo.builder() + .tenantId(tenantId) + .myPartition(true) + .build(); + + final FutureCallback emptyCallback = new FutureCallback<>() { + @Override + public void onSuccess(Void result) {} + + @Override + public void onFailure(@NonNull Throwable t) {} + }; + + ExecutorService wsCallBackExecutor; + ExecutorService tsCallBackExecutor; + + @Mock + TbClusterService clusterService; + @Mock + PartitionService partitionService; + @Mock + SubscriptionManagerService subscriptionManagerService; + @Mock + AttributesService attrService; + @Mock + TimeseriesService tsService; + @Mock + TbEntityViewService tbEntityViewService; + @Mock + TbApiUsageReportClient apiUsageClient; + @Mock + TbApiUsageStateService apiUsageStateService; + + DefaultTelemetrySubscriptionService telemetryService; + + @BeforeEach + void setup() { + telemetryService = new DefaultTelemetrySubscriptionService(attrService, tsService, tbEntityViewService, apiUsageClient, apiUsageStateService); + ReflectionTestUtils.setField(telemetryService, "clusterService", clusterService); + ReflectionTestUtils.setField(telemetryService, "partitionService", partitionService); + ReflectionTestUtils.setField(telemetryService, "subscriptionManagerService", Optional.of(subscriptionManagerService)); + + wsCallBackExecutor = MoreExecutors.newDirectExecutorService(); + ReflectionTestUtils.setField(telemetryService, "wsCallBackExecutor", wsCallBackExecutor); + + tsCallBackExecutor = MoreExecutors.newDirectExecutorService(); + ReflectionTestUtils.setField(telemetryService, "tsCallBackExecutor", tsCallBackExecutor); + + apiUsageState = new ApiUsageState(); + apiUsageState.setDbStorageState(ApiUsageStateValue.ENABLED); + lenient().when(apiUsageStateService.getApiUsageState(tenantId)).thenReturn(apiUsageState); + + lenient().when(partitionService.resolve(ServiceType.TB_CORE, tenantId, entityId)).thenReturn(tpi); + + lenient().when(tsService.save(tenantId, entityId, sampleTelemetry, sampleTtl)).thenReturn(immediateFuture(sampleTelemetry.size())); + lenient().when(tsService.saveWithoutLatest(tenantId, entityId, sampleTelemetry, sampleTtl)).thenReturn(immediateFuture(sampleTelemetry.size())); + lenient().when(tsService.saveLatest(tenantId, entityId, sampleTelemetry)).thenReturn(immediateFuture(listOfNNumbers(sampleTelemetry.size()))); + + // mock no entity views + lenient().when(tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId)).thenReturn(immediateFuture(Collections.emptyList())); + + // send partition change event so currentPartitions set is populated + telemetryService.onTbApplicationEvent(new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of(new QueueKey(ServiceType.TB_CORE), Set.of(tpi)))); + } + + @AfterEach + void cleanup() { + wsCallBackExecutor.shutdownNow(); + tsCallBackExecutor.shutdownNow(); + } + + @Test + void shouldReportStorageDataPointsApiUsageWhenTimeSeriesIsSaved() { + // GIVEN + var request = TimeseriesSaveRequest.builder() + .tenantId(tenantId) + .customerId(customerId) + .entityId(entityId) + .entries(sampleTelemetry) + .ttl(sampleTtl) + .saveTimeseries(true) + .saveLatest(false) + .sendWsUpdate(false) + .callback(emptyCallback) + .build(); + + // WHEN + telemetryService.saveTimeseries(request); + + // THEN + then(apiUsageClient).should().report(tenantId, customerId, ApiUsageRecordKey.STORAGE_DP_COUNT, sampleTelemetry.size()); + } + + @Test + void shouldNotReportStorageDataPointsApiUsageWhenTimeSeriesIsNotSaved() { + // GIVEN + var request = TimeseriesSaveRequest.builder() + .tenantId(tenantId) + .customerId(customerId) + .entityId(entityId) + .entries(sampleTelemetry) + .ttl(sampleTtl) + .saveTimeseries(false) + .saveLatest(true) + .sendWsUpdate(true) + .callback(emptyCallback) + .build(); + + // WHEN + telemetryService.saveTimeseries(request); + + // THEN + then(apiUsageClient).shouldHaveNoInteractions(); + } + + @Test + void shouldThrowStorageDisabledWhenTimeSeriesIsSavedAndStorageIsDisabled() { + // GIVEN + apiUsageState.setDbStorageState(ApiUsageStateValue.DISABLED); + + SettableFuture future = SettableFuture.create(); + var request = TimeseriesSaveRequest.builder() + .tenantId(tenantId) + .customerId(customerId) + .entityId(entityId) + .entries(sampleTelemetry) + .ttl(sampleTtl) + .saveTimeseries(true) + .saveLatest(true) + .sendWsUpdate(true) + .future(future) + .build(); + + // WHEN + telemetryService.saveTimeseries(request); + + // THEN + assertThat(future).failsWithin(Duration.ofSeconds(5)) + .withThrowableOfType(ExecutionException.class) + .withCauseInstanceOf(RuntimeException.class) + .withMessageContaining("DB storage writes are disabled due to API limits!"); + } + + @Test + void shouldNotThrowStorageDisabledWhenTimeSeriesIsNotSavedAndStorageIsDisabled() { + // GIVEN + apiUsageState.setDbStorageState(ApiUsageStateValue.DISABLED); + + SettableFuture future = SettableFuture.create(); + var request = TimeseriesSaveRequest.builder() + .tenantId(tenantId) + .customerId(customerId) + .entityId(entityId) + .entries(sampleTelemetry) + .ttl(sampleTtl) + .saveTimeseries(false) + .saveLatest(true) + .sendWsUpdate(true) + .future(future) + .build(); + + // WHEN + telemetryService.saveTimeseries(request); + + // THEN + assertThat(future).succeedsWithin(Duration.ofSeconds(5)); + } + + @Test + void shouldCopyLatestToEntityViewWhenLatestIsSavedOnMainEntity() { + // GIVEN + var entityView = new EntityView(new EntityViewId(UUID.randomUUID())); + entityView.setTenantId(tenantId); + entityView.setCustomerId(customerId); + entityView.setEntityId(entityId); + entityView.setKeys(new TelemetryEntityView(sampleTelemetry.stream().map(KvEntry::getKey).toList(), new AttributesEntityView())); + + // mock that there is one entity view + given(tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId)).willReturn(immediateFuture(List.of(entityView))); + // mock that save latest call for entity view is successful + given(tsService.saveLatest(tenantId, entityView.getId(), sampleTelemetry)).willReturn(immediateFuture(listOfNNumbers(sampleTelemetry.size()))); + // mock TPI for entity view + given(partitionService.resolve(ServiceType.TB_CORE, tenantId, entityView.getId())).willReturn(tpi); + + var request = TimeseriesSaveRequest.builder() + .tenantId(tenantId) + .customerId(customerId) + .entityId(entityId) + .entries(sampleTelemetry) + .ttl(sampleTtl) + .saveTimeseries(false) + .saveLatest(true) + .sendWsUpdate(false) + .callback(emptyCallback) + .build(); + + // WHEN + telemetryService.saveTimeseries(request); + + // THEN + // should save latest to both the main entity and it's entity view + then(tsService).should().saveLatest(tenantId, entityId, sampleTelemetry); + then(tsService).should().saveLatest(tenantId, entityView.getId(), sampleTelemetry); + then(tsService).shouldHaveNoMoreInteractions(); + + // should send WS update only for entity view (WS update for the main entity is disabled in the save request) + then(subscriptionManagerService).should().onTimeSeriesUpdate(tenantId, entityView.getId(), sampleTelemetry, TbCallback.EMPTY); + then(subscriptionManagerService).shouldHaveNoMoreInteractions(); + } + + @Test + void shouldNotCopyLatestToEntityViewWhenLatestIsNotSavedOnMainEntity() { + // GIVEN + var request = TimeseriesSaveRequest.builder() + .tenantId(tenantId) + .customerId(customerId) + .entityId(entityId) + .entries(sampleTelemetry) + .ttl(sampleTtl) + .saveTimeseries(true) + .saveLatest(false) + .sendWsUpdate(false) + .callback(emptyCallback) + .build(); + + // WHEN + telemetryService.saveTimeseries(request); + + // THEN + // should save only time series for the main entity + then(tsService).should().saveWithoutLatest(tenantId, entityId, sampleTelemetry, sampleTtl); + then(tsService).shouldHaveNoMoreInteractions(); + + // should not send any WS updates + then(subscriptionManagerService).shouldHaveNoInteractions(); + } + + @ParameterizedTest + @MethodSource("booleanCombinations") + void shouldCallCorrectApiBasedOnBooleanFlagsInTheSaveRequest(boolean saveTimeseries, boolean saveLatest, boolean sendWsUpdate) { + // GIVEN + var request = TimeseriesSaveRequest.builder() + .tenantId(tenantId) + .customerId(customerId) + .entityId(entityId) + .entries(sampleTelemetry) + .ttl(sampleTtl) + .saveTimeseries(saveTimeseries) + .saveLatest(saveLatest) + .sendWsUpdate(sendWsUpdate) + .callback(emptyCallback) + .build(); + + // WHEN + telemetryService.saveTimeseries(request); + + // THEN + if (saveTimeseries && saveLatest) { + then(tsService).should().save(tenantId, entityId, sampleTelemetry, sampleTtl); + } else if (saveLatest) { + then(tsService).should().saveLatest(tenantId, entityId, sampleTelemetry); + } else if (saveTimeseries) { + then(tsService).should().saveWithoutLatest(tenantId, entityId, sampleTelemetry, sampleTtl); + } + then(tsService).shouldHaveNoMoreInteractions(); + + if (sendWsUpdate) { + then(subscriptionManagerService).should().onTimeSeriesUpdate(tenantId, entityId, sampleTelemetry, TbCallback.EMPTY); + } else { + then(subscriptionManagerService).shouldHaveNoInteractions(); + } + } + + private static Stream booleanCombinations() { + return Stream.of( + Arguments.of(true, true, true), + Arguments.of(true, true, false), + Arguments.of(true, false, true), + Arguments.of(true, false, false), + Arguments.of(false, true, true), + Arguments.of(false, true, false), + Arguments.of(false, false, true), + Arguments.of(false, false, false) + ); + } + + // used to emulate sequence numbers returned by save latest API + private static List listOfNNumbers(int N) { + return LongStream.range(0, N).boxed().toList(); + } + +} diff --git a/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java b/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java new file mode 100644 index 0000000000..cf05f9905b --- /dev/null +++ b/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.api; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class TimeseriesSaveRequestTest { + + @Test + void testBooleanFlagsDefaultToTrue() { + var request = TimeseriesSaveRequest.builder().build(); + + assertThat(request.isSaveTimeseries()).isTrue(); + assertThat(request.isSaveLatest()).isTrue(); + assertThat(request.isSendWsUpdate()).isTrue(); + } + +} From cad35a969e3db4894554f24cb72797b0356ba621 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Wed, 15 Jan 2025 17:48:53 +0200 Subject: [PATCH 07/30] Save time series strategies: add tests for persistence strategies --- .../DeduplicatePersistenceStrategyTest.java | 147 ++++++++++++++++++ ...OnEveryMessagePersistenceStrategyTest.java | 48 ++++++ .../strategy/PersistenceStrategyTest.java | 55 +++++++ .../strategy/SkipPersistenceStrategyTest.java | 48 ++++++ 4 files changed, 298 insertions(+) create mode 100644 rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java create mode 100644 rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/OnEveryMessagePersistenceStrategyTest.java create mode 100644 rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/PersistenceStrategyTest.java create mode 100644 rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/SkipPersistenceStrategyTest.java diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java new file mode 100644 index 0000000000..3f57a7f3df --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java @@ -0,0 +1,147 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.telemetry.strategy; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class DeduplicatePersistenceStrategyTest { + + final int deduplicationIntervalSecs = 10; + + DeduplicatePersistenceStrategy strategy; + + @BeforeEach + void setup() { + strategy = new DeduplicatePersistenceStrategy(deduplicationIntervalSecs); + } + + @Test + void shouldThrowWhenDeduplicationIntervalIsLessThanOneSecond() { + assertThatThrownBy(() -> new DeduplicatePersistenceStrategy(0)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Deduplication interval must be at least 1 second(s), was 0 second(s)"); + } + + @Test + void shouldReturnTrueForFirstCallInInterval() { + long ts = 1_000_000L; + UUID originator = UUID.randomUUID(); + + assertThat(strategy.shouldPersist(ts, originator)).isTrue(); + } + + @Test + void shouldReturnFalseForSubsequentCallsInInterval() { + long baseTs = 1_000_000L; + UUID originator = UUID.randomUUID(); + + // Initial call should return true + assertThat(strategy.shouldPersist(baseTs, originator)).isTrue(); + + // Subsequent call within the same interval should return false for the same originator + long withinSameIntervalTs = baseTs + 1000L; + assertThat(strategy.shouldPersist(withinSameIntervalTs, originator)).isFalse(); + } + + @Test + void shouldHandleMultipleOriginatorsIndependently() { + long baseTs = 1_000_000L; + UUID originator1 = UUID.randomUUID(); + UUID originator2 = UUID.randomUUID(); + + // First call for different originators in the same interval should return true independently + assertThat(strategy.shouldPersist(baseTs, originator1)).isTrue(); + assertThat(strategy.shouldPersist(baseTs, originator2)).isTrue(); + + // Subsequent calls for the same originators within the same interval should return false + assertThat(strategy.shouldPersist(baseTs + 500L, originator1)).isFalse(); + assertThat(strategy.shouldPersist(baseTs + 500L, originator2)).isFalse(); + } + + @Test + void shouldHandleEdgeCaseTimestamps() { + long minTs = Long.MIN_VALUE; + long maxTs = Long.MAX_VALUE; + UUID originator = UUID.randomUUID(); + + assertThat(strategy.shouldPersist(minTs, originator)).isTrue(); + assertThat(strategy.shouldPersist(minTs + 1L, originator)).isFalse(); + + assertThat(strategy.shouldPersist(maxTs, originator)).isTrue(); + assertThat(strategy.shouldPersist(maxTs - 1L, originator)).isFalse(); + } + + @Test + void shouldResetDeduplicationAtIntervalBoundaries() { + UUID originator = UUID.randomUUID(); + + // check 1st interval + long firstIntervalStart = 0L; + long firstIntervalEnd = firstIntervalStart + Duration.ofSeconds(deduplicationIntervalSecs).toMillis() - 1L; + long firstIntervalMiddle = calculateMiddle(firstIntervalStart, firstIntervalEnd); + + assertThat(strategy.shouldPersist(firstIntervalStart, originator)).isTrue(); + assertThat(strategy.shouldPersist(firstIntervalStart + 1, originator)).isFalse(); + assertThat(strategy.shouldPersist(firstIntervalMiddle, originator)).isFalse(); + assertThat(strategy.shouldPersist(firstIntervalEnd - 1, originator)).isFalse(); + assertThat(strategy.shouldPersist(firstIntervalEnd, originator)).isFalse(); + + // check 2nd interval + long secondIntervalStart = firstIntervalEnd + 1L; + long secondIntervalEnd = secondIntervalStart + Duration.ofSeconds(deduplicationIntervalSecs).toMillis() - 1L; + long secondIntervalMiddle = calculateMiddle(secondIntervalStart, secondIntervalEnd); + + assertThat(strategy.shouldPersist(secondIntervalStart, originator)).isTrue(); + assertThat(strategy.shouldPersist(secondIntervalStart + 1, originator)).isFalse(); + assertThat(strategy.shouldPersist(secondIntervalMiddle, originator)).isFalse(); + assertThat(strategy.shouldPersist(secondIntervalEnd - 1, originator)).isFalse(); + assertThat(strategy.shouldPersist(secondIntervalEnd, originator)).isFalse(); + } + + @Test + void shouldHandleMultipleOriginatorsOverMultipleIntervals() { + UUID originator1 = UUID.randomUUID(); + UUID originator2 = UUID.randomUUID(); + long baseTs = 0L; + + // First interval for both originators + assertThat(strategy.shouldPersist(baseTs, originator1)).isTrue(); + assertThat(strategy.shouldPersist(baseTs, originator2)).isTrue(); + + // Move to the next interval + long nextIntervalTs = baseTs + Duration.ofSeconds(10).toMillis(); + + // Each originator should be allowed again in the new interval + assertThat(strategy.shouldPersist(nextIntervalTs, originator1)).isTrue(); + assertThat(strategy.shouldPersist(nextIntervalTs, originator2)).isTrue(); + + // Subsequent calls in the same new interval should return false + assertThat(strategy.shouldPersist(nextIntervalTs + 500L, originator1)).isFalse(); + assertThat(strategy.shouldPersist(nextIntervalTs + 500L, originator2)).isFalse(); + } + + private static long calculateMiddle(long start, long end) { + return start + (end - start) / 2; + } + +} diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/OnEveryMessagePersistenceStrategyTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/OnEveryMessagePersistenceStrategyTest.java new file mode 100644 index 0000000000..125da3a495 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/OnEveryMessagePersistenceStrategyTest.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.telemetry.strategy; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.UUID; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +class OnEveryMessagePersistenceStrategyTest { + + @ParameterizedTest + @MethodSource("edgeCaseProvider") + void shouldAlwaysReturnTrueForAnyInput(long timestamp, UUID originator) { + var onEveryMessage = OnEveryMessagePersistenceStrategy.getInstance(); + assertThat(onEveryMessage.shouldPersist(timestamp, originator)).isTrue(); + } + + private static Stream edgeCaseProvider() { + return Stream.of( + Arguments.of(Long.MIN_VALUE, new UUID(0L, 0L)), + Arguments.of(Long.MAX_VALUE, new UUID(Long.MAX_VALUE, Long.MAX_VALUE)), + Arguments.of(0L, new UUID(0L, 0L)), + Arguments.of(-1L, new UUID(-1L, -1L)), + Arguments.of(1L, new UUID(1L, 1L)), + Arguments.of(42L, UUID.randomUUID()), + Arguments.of(1000L, null) + ); + } + +} diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/PersistenceStrategyTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/PersistenceStrategyTest.java new file mode 100644 index 0000000000..71a8eabed5 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/PersistenceStrategyTest.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.telemetry.strategy; + +import org.junit.jupiter.api.Test; +import org.springframework.test.util.ReflectionTestUtils; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; + +class PersistenceStrategyTest { + + @Test + void testOnEveryMessageReturnsCorrectInstance() { + PersistenceStrategy strategy = PersistenceStrategy.onEveryMessage(); + assertThat(strategy) + .isNotNull() + .isInstanceOf(OnEveryMessagePersistenceStrategy.class); + } + + @Test + void testDeduplicateReturnsCorrectInstance() { + int validDeduplicationIntervalSecs = 5; + PersistenceStrategy strategy = PersistenceStrategy.deduplicate(validDeduplicationIntervalSecs); + assertThat(strategy) + .isNotNull() + .isInstanceOf(DeduplicatePersistenceStrategy.class); + + long actualDeduplicationIntervalMillis = (long) ReflectionTestUtils.getField(strategy, "deduplicationIntervalMillis"); + assertThat(actualDeduplicationIntervalMillis).isEqualTo(Duration.ofSeconds(validDeduplicationIntervalSecs).toMillis()); + } + + @Test + void testSkipReturnsCorrectInstance() { + PersistenceStrategy strategy = PersistenceStrategy.skip(); + assertThat(strategy) + .isNotNull() + .isInstanceOf(SkipPersistenceStrategy.class); + } + +} diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/SkipPersistenceStrategyTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/SkipPersistenceStrategyTest.java new file mode 100644 index 0000000000..1a63ce7460 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/SkipPersistenceStrategyTest.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.telemetry.strategy; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.UUID; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +class SkipPersistenceStrategyTest { + + @ParameterizedTest + @MethodSource("edgeCaseProvider") + void shouldAlwaysReturnFalseForAnyInput(long timestamp, UUID originator) { + var skipStrategy = SkipPersistenceStrategy.getInstance(); + assertThat(skipStrategy.shouldPersist(timestamp, originator)).isFalse(); + } + + private static Stream edgeCaseProvider() { + return Stream.of( + Arguments.of(Long.MIN_VALUE, new UUID(0L, 0L)), + Arguments.of(Long.MAX_VALUE, new UUID(Long.MAX_VALUE, Long.MAX_VALUE)), + Arguments.of(0L, new UUID(0L, 0L)), + Arguments.of(-1L, new UUID(-1L, -1L)), + Arguments.of(1L, new UUID(1L, 1L)), + Arguments.of(42L, UUID.randomUUID()), + Arguments.of(1000L, null) + ); + } + +} From 3aac81745f5d52f1d26ac2e679ed8d040e14f9cb Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Thu, 16 Jan 2025 12:58:22 +0200 Subject: [PATCH 08/30] Save time series strategies: add test for changes in entity view service --- .../DefaultTbEntityViewServiceTest.java | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java new file mode 100644 index 0000000000..b36890f775 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java @@ -0,0 +1,109 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.entityview; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.objects.AttributesEntityView; +import org.thingsboard.server.common.data.objects.TelemetryEntityView; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.entityview.EntityViewService; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; + +import java.util.List; +import java.util.UUID; + +import static com.google.common.util.concurrent.Futures.immediateFuture; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.then; + +@ExtendWith(MockitoExtension.class) +class DefaultTbEntityViewServiceTest { + + final TenantId tenantId = TenantId.fromUUID(UUID.fromString("f09c8180-686c-11ef-9471-a71d33080e9c")); + final EntityId entityId = DeviceId.fromString("782aaab0-c7a8-11ef-a668-79582e785d5f"); + + @Mock + EntityViewService entityViewService; + @Mock + AttributesService attributesService; + @Mock + TelemetrySubscriptionService tsSubService; + @Mock + TimeseriesService tsService; + + DefaultTbEntityViewService defaultTbEntityViewService; + + @BeforeEach + void setup() { + defaultTbEntityViewService = new DefaultTbEntityViewService(entityViewService, attributesService, tsSubService, tsService); + } + + @Test + void shouldNotSaveTimeseriesWhenCopyingLatestToEntityView() throws Exception { + // GIVEN + var entityView = new EntityView(new EntityViewId(UUID.randomUUID())); + entityView.setTenantId(tenantId); + entityView.setEntityId(entityId); + entityView.setKeys(new TelemetryEntityView(List.of("temperature"), new AttributesEntityView())); + + List latest = List.of(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))); + + given(tsService.findAll(eq(tenantId), eq(entityId), anyList())).willReturn(immediateFuture(latest)); + + // WHEN + defaultTbEntityViewService.updateEntityViewAttributes(tenantId, entityView, null, null); + + // THEN + var captor = ArgumentCaptor.forClass(TimeseriesSaveRequest.class); + then(tsSubService).should().saveTimeseries(captor.capture()); + + var expectedCopyLatestRequest = TimeseriesSaveRequest.builder() + .tenantId(tenantId) + .entityId(entityView.getId()) + .entries(latest) + .ttl(0L) + .saveTimeseries(false) + .saveLatest(true) + .sendWsUpdate(true) + .build(); + + var actualCopyLatestRequest = captor.getValue(); + + assertThat(actualCopyLatestRequest) + .usingRecursiveComparison() + .ignoringFields("callback") + .isEqualTo(expectedCopyLatestRequest); + } + +} From 0b277661bdba594b2d22b04914a30cde039b1c23 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Fri, 17 Jan 2025 12:36:52 +0200 Subject: [PATCH 09/30] Save time series strategies: add tests for changes in save time series node --- .../engine/telemetry/TbMsgTimeseriesNode.java | 6 +- .../TbMsgTimeseriesNodeConfiguration.java | 11 +- .../DeduplicatePersistenceStrategy.java | 5 + .../telemetry/TbMsgTimeseriesNodeTest.java | 365 ++++++++++++++++-- 4 files changed, 342 insertions(+), 45 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index db2421e307..422065f81f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -90,11 +90,11 @@ public class TbMsgTimeseriesNode implements TbNode { onTenantProfileUpdate(ctx.getTenantProfile()); persistenceSettings = config.getPersistenceSettings(); if (persistenceSettings == null) { - throw new TbNodeException("Persistence settings cannot be null!", true); + throw new TbNodeException("Persistence settings cannot be null", true); } } - void onTenantProfileUpdate(TenantProfile tenantProfile) { + private void onTenantProfileUpdate(TenantProfile tenantProfile) { DefaultTenantProfileConfiguration configuration = (DefaultTenantProfileConfiguration) tenantProfile.getProfileData().getConfiguration(); tenantProfileDefaultStorageTtl = TimeUnit.DAYS.toSeconds(configuration.getDefaultStorageTtlDays()); } @@ -193,7 +193,7 @@ public class TbMsgTimeseriesNode implements TbNode { boolean hasChanges = false; switch (fromVersion) { case 0: - if (oldConfiguration.has("persistenceSettings")) { + if (oldConfiguration.has("persistenceSettings") && !oldConfiguration.has("skipLatestPersistence")) { break; } hasChanges = true; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java index 8be8cac24e..f702ecc6da 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java @@ -16,6 +16,7 @@ package org.thingsboard.rule.engine.telemetry; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -67,11 +68,15 @@ public class TbMsgTimeseriesNodeConfiguration implements NodeConfiguration Sets.newConcurrentHashSet()); } + @JsonProperty("deduplicationIntervalSecs") + public long getDeduplicationIntervalSecs() { + return Duration.ofMillis(deduplicationIntervalMillis).toSeconds(); + } + @Override public boolean shouldPersist(long ts, UUID originatorUuid) { long intervalNumber = ts / deduplicationIntervalMillis; diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java index 71d060aab2..d0727b1a3c 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java @@ -41,6 +41,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; @@ -57,25 +58,30 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.assertArg; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.then; +import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("c8f34868-603a-4433-876a-7d356e5cf377")); private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("e5095e9a-04f4-44c9-b443-1cf1b97d3384")); - private final TenantProfileId TENANT_PROFILE_ID = new TenantProfileId(UUID.fromString("ab78dd78-83d0-43fa-869f-d42ec9ed1744")); + + private TenantProfile tenantProfile; private TbMsgTimeseriesNode node; private TbMsgTimeseriesNodeConfiguration config; - private long tenantProfileDefaultStorageTtl; @Mock private TbContext ctxMock; @@ -84,6 +90,17 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { @BeforeEach public void setUp() throws TbNodeException { + tenantProfile = new TenantProfile(new TenantProfileId(UUID.fromString("ab78dd78-83d0-43fa-869f-d42ec9ed1744"))); + var tenantProfileConfiguration = new DefaultTenantProfileConfiguration(); + tenantProfileConfiguration.setDefaultStorageTtlDays(5); + var tenantProfileData = new TenantProfileData(); + tenantProfileData.setConfiguration(tenantProfileConfiguration); + tenantProfile.setProfileData(tenantProfileData); + lenient().when(ctxMock.getTenantProfile()).thenReturn(tenantProfile); + + lenient().when(ctxMock.getTenantId()).thenReturn(TENANT_ID); + lenient().when(ctxMock.getTelemetryService()).thenReturn(telemetryServiceMock); + node = spy(new TbMsgTimeseriesNode()); config = new TbMsgTimeseriesNodeConfiguration().defaultConfiguration(); } @@ -95,18 +112,47 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { assertThat(config.isUseServerTs()).isFalse(); } + @Test + public void whenInit_thenShouldAddTenantProfileListener() throws Exception { + // GIVEN-WHEN + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + // THEN + then(ctxMock).should().addTenantProfileListener(any()); + } + + @Test + public void givenPersistenceSettingsAreNull_whenInit_thenThrowsException() { + // GIVEN + config.setPersistenceSettings(null); + + // WHEN-THEN + assertThatThrownBy(() -> node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config)))) + .isInstanceOf(TbNodeException.class) + .matches(e -> ((TbNodeException) e).isUnrecoverable()) + .hasMessage("Persistence settings cannot be null"); + } + @ParameterizedTest @EnumSource(TbMsgType.class) public void givenMsgTypeAndEmptyMsgData_whenOnMsg_thenVerifyFailureMsg(TbMsgType msgType) throws TbNodeException { - init(); + // GIVEN + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + TbMsg msg = TbMsg.newMsg() .type(msgType) .originator(DEVICE_ID) .copyMetaData(TbMsgMetaData.EMPTY) .data(TbMsg.EMPTY_JSON_ARRAY) .build(); + + // WHEN node.onMsg(ctxMock, msg); + // THEN + then(ctxMock).should().addTenantProfileListener(any()); + then(ctxMock).should().getTenantProfile(); + ArgumentCaptor throwableCaptor = ArgumentCaptor.forClass(Throwable.class); verify(ctxMock).tellFailure(eq(msg), throwableCaptor.capture()); @@ -120,9 +166,11 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { } @Test - public void givenTtlFromConfigIsZeroAndUseServiceTsIsTrue_whenOnMsg_thenSaveTimeseriesUsingTenantProfileDefaultTtl() throws TbNodeException { + public void givenTtlFromConfigIsZeroAndUseServerTsIsTrue_whenOnMsg_thenSaveTimeseriesUsingTenantProfileDefaultTtl() throws TbNodeException { + // GIVEN config.setUseServerTs(true); - init(); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); String data = """ { @@ -137,23 +185,28 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .data(data) .build(); - when(ctxMock.getTelemetryService()).thenReturn(telemetryServiceMock); - when(ctxMock.getTenantId()).thenReturn(TENANT_ID); doAnswer(invocation -> { TimeseriesSaveRequest request = invocation.getArgument(0); request.getCallback().onSuccess(null); return null; }).when(telemetryServiceMock).saveTimeseries(any(TimeseriesSaveRequest.class)); + // WHEN node.onMsg(ctxMock, msg); + // THEN + then(ctxMock).should().getTenantId(); + then(ctxMock).should().getTelemetryService(); + then(ctxMock).should().addTenantProfileListener(any()); + then(ctxMock).should().getTenantProfile(); + List expectedList = getTsKvEntriesListWithTs(data, System.currentTimeMillis()); verify(telemetryServiceMock).saveTimeseries(assertArg(request -> { assertThat(request.getTenantId()).isEqualTo(TENANT_ID); assertThat(request.getCustomerId()).isNull(); assertThat(request.getEntityId()).isEqualTo(DEVICE_ID); assertThat(request.getEntries()).usingRecursiveFieldByFieldElementComparatorIgnoringFields("ts").containsExactlyElementsOf(expectedList); - assertThat(request.getTtl()).isEqualTo(tenantProfileDefaultStorageTtl); + assertThat(request.getTtl()).isEqualTo(extractTtlAsSeconds(tenantProfile)); assertThat(request.isSaveLatest()).isTrue(); assertThat(request.getCallback()).isInstanceOf(TelemetryNodeCallback.class); })); @@ -162,9 +215,9 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { } @Test - public void givenSkipLatestPersistenceIsTrueAndTtlFromConfig_whenOnMsg_thenSaveTimeseriesUsingTtlFromConfig() throws TbNodeException { - long ttlFromConfig = 5L; - config.setDefaultTTL(ttlFromConfig); + public void givenSkipLatestPersistenceSettingsAndTtlFromConfig_whenOnMsg_thenSaveTimeseriesUsingTtlFromConfig() throws TbNodeException { + // GIVEN + config.setDefaultTTL(10L); var timeseriesStrategy = PersistenceStrategy.onEveryMessage(); var latestStrategy = PersistenceStrategy.skip(); @@ -172,7 +225,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { var persistenceSettings = new TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Advanced(timeseriesStrategy, latestStrategy, webSockets); config.setPersistenceSettings(persistenceSettings); - init(); + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); String data = """ { @@ -189,23 +242,28 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .data(data) .build(); - when(ctxMock.getTelemetryService()).thenReturn(telemetryServiceMock); - when(ctxMock.getTenantId()).thenReturn(TENANT_ID); doAnswer(invocation -> { TimeseriesSaveRequest request = invocation.getArgument(0); request.getCallback().onSuccess(null); return null; }).when(telemetryServiceMock).saveTimeseries(any(TimeseriesSaveRequest.class)); + // WHEN node.onMsg(ctxMock, msg); + // THEN + then(ctxMock).should().getTenantId(); + then(ctxMock).should().getTelemetryService(); + then(ctxMock).should().addTenantProfileListener(any()); + then(ctxMock).should().getTenantProfile(); + List expectedList = getTsKvEntriesListWithTs(data, ts); verify(telemetryServiceMock).saveTimeseries(assertArg(request -> { assertThat(request.getTenantId()).isEqualTo(TENANT_ID); assertThat(request.getCustomerId()).isNull(); assertThat(request.getEntityId()).isEqualTo(DEVICE_ID); assertThat(request.getEntries()).containsExactlyElementsOf(expectedList); - assertThat(request.getTtl()).isEqualTo(ttlFromConfig); + assertThat(request.getTtl()).isEqualTo(config.getDefaultTTL()); assertThat(request.isSaveTimeseries()).isTrue(); assertThat(request.isSaveLatest()).isFalse(); assertThat(request.isSendWsUpdate()).isTrue(); @@ -218,11 +276,10 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { @ParameterizedTest @MethodSource public void givenTtlFromConfigAndTtlFromMd_whenOnMsg_thenVerifyTtl(String ttlFromMd, long ttlFromConfig, long expectedTtl) throws TbNodeException { + // GIVEN config.setDefaultTTL(ttlFromConfig); - init(); - when(ctxMock.getTelemetryService()).thenReturn(telemetryServiceMock); - when(ctxMock.getTenantId()).thenReturn(TENANT_ID); + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); String data = """ { @@ -238,8 +295,11 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .copyMetaData(metadata) .data(data) .build(); + + // WHEN node.onMsg(ctxMock, msg); + // THEN verify(telemetryServiceMock).saveTimeseries(assertArg(request -> { assertThat(request.getTenantId()).isEqualTo(TENANT_ID); assertThat(request.getCustomerId()).isNull(); @@ -262,26 +322,6 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { ); } - private void init() throws TbNodeException { - var configuration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); - var tenantProfile = getTenantProfile(); - when(ctxMock.getTenantProfile()).thenReturn(tenantProfile); - tenantProfile.getProfileConfiguration().ifPresent(profileConfiguration -> - tenantProfileDefaultStorageTtl = TimeUnit.DAYS.toSeconds(profileConfiguration.getDefaultStorageTtlDays())); - node.init(ctxMock, configuration); - verify(ctxMock).addTenantProfileListener(any()); - } - - private TenantProfile getTenantProfile() { - var tenantProfile = new TenantProfile(TENANT_PROFILE_ID); - var tenantProfileData = new TenantProfileData(); - var tenantProfileConfiguration = new DefaultTenantProfileConfiguration(); - tenantProfileConfiguration.setDefaultStorageTtlDays(5); - tenantProfileData.setConfiguration(tenantProfileConfiguration); - tenantProfile.setProfileData(tenantProfileData); - return tenantProfile; - } - private static List getTsKvEntriesListWithTs(String data, long ts) { Map> tsKvMap = JsonConverter.convertToTelemetry(JsonParser.parseString(data), ts); List expectedList = new ArrayList<>(); @@ -293,6 +333,253 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { return expectedList; } + @Test + public void givenOnEveryMessagePersistenceSettingsAndSameMessageTwoTimes_whenOnMsg_thenPersistSameMessageTwoTimes() throws TbNodeException { + // GIVEN + config.setPersistenceSettings(new TbMsgTimeseriesNodeConfiguration.PersistenceSettings.OnEveryMessage()); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(DEVICE_ID) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", "123"))) + .build(); + + // WHEN-THEN + var expectedSaveRequest = TimeseriesSaveRequest.builder() + .tenantId(TENANT_ID) + .customerId(msg.getCustomerId()) + .entityId(msg.getOriginator()) + .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) + .ttl(extractTtlAsSeconds(tenantProfile)) + .saveTimeseries(true) + .saveLatest(true) + .sendWsUpdate(true) + .build(); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(times(1)).saveTimeseries(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest).usingRecursiveComparison().ignoringFields("callback").isEqualTo(expectedSaveRequest) + )); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(times(2)).saveTimeseries(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest).usingRecursiveComparison().ignoringFields("callback").isEqualTo(expectedSaveRequest) + )); + } + + @Test + public void givenDeduplicatePersistenceSettingsAndSameMessageTwoTimes_whenOnMsg_thenPersistThisMessageOnlyFirstTime() throws TbNodeException { + // GIVEN + config.setPersistenceSettings(new TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Deduplicate(10)); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(DEVICE_ID) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", "123"))) + .build(); + + // WHEN-THEN + var expectedSaveRequest = TimeseriesSaveRequest.builder() + .tenantId(TENANT_ID) + .customerId(msg.getCustomerId()) + .entityId(msg.getOriginator()) + .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) + .ttl(extractTtlAsSeconds(tenantProfile)) + .saveTimeseries(true) + .saveLatest(true) + .sendWsUpdate(true) + .build(); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should().saveTimeseries(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest).usingRecursiveComparison().ignoringFields("callback").isEqualTo(expectedSaveRequest) + )); + + clearInvocations(telemetryServiceMock, ctxMock); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(never()).saveTimeseries(any()); + } + + @Test + public void givenWebsocketsOnlyPersistenceSettingsAndSameMessageTwoTimes_whenOnMsg_thenSendsOnlyWsUpdateTwoTimes() throws TbNodeException { + // GIVEN + config.setPersistenceSettings(new TbMsgTimeseriesNodeConfiguration.PersistenceSettings.WebSocketsOnly()); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(DEVICE_ID) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", "123"))) + .build(); + + // WHEN-THEN + var expectedSaveRequest = TimeseriesSaveRequest.builder() + .tenantId(TENANT_ID) + .customerId(msg.getCustomerId()) + .entityId(msg.getOriginator()) + .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) + .ttl(extractTtlAsSeconds(tenantProfile)) + .saveTimeseries(false) + .saveLatest(false) + .sendWsUpdate(true) + .build(); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(times(1)).saveTimeseries(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest).usingRecursiveComparison().ignoringFields("callback").isEqualTo(expectedSaveRequest) + )); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(times(2)).saveTimeseries(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest).usingRecursiveComparison().ignoringFields("callback").isEqualTo(expectedSaveRequest) + )); + } + + @Test + public void givenAdvancedPersistenceSettingsWithOnEveryMessageStrategiesForAllActionsAndSameMessageTwoTimes_whenOnMsg_thenPersistSameMessageTwoTimes() throws TbNodeException { + // GIVEN + config.setPersistenceSettings(new TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Advanced( + PersistenceStrategy.onEveryMessage(), + PersistenceStrategy.onEveryMessage(), + PersistenceStrategy.onEveryMessage() + )); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(DEVICE_ID) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", "123"))) + .build(); + + // WHEN-THEN + var expectedSaveRequest = TimeseriesSaveRequest.builder() + .tenantId(TENANT_ID) + .customerId(msg.getCustomerId()) + .entityId(msg.getOriginator()) + .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) + .ttl(extractTtlAsSeconds(tenantProfile)) + .saveTimeseries(true) + .saveLatest(true) + .sendWsUpdate(true) + .build(); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(times(1)).saveTimeseries(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest).usingRecursiveComparison().ignoringFields("callback").isEqualTo(expectedSaveRequest) + )); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(times(2)).saveTimeseries(assertArg( + actualSaveRequest -> assertThat(actualSaveRequest).usingRecursiveComparison().ignoringFields("callback").isEqualTo(expectedSaveRequest) + )); + } + + @Test + public void givenAdvancedPersistenceSettingsWithDifferentDeduplicateStrategyForEachAction_whenOnMsg_thenEvaluatesStrategiesForEachActionsIndependently() throws TbNodeException { + // GIVEN + config.setPersistenceSettings(new TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Advanced( + PersistenceStrategy.deduplicate(1), + PersistenceStrategy.deduplicate(2), + PersistenceStrategy.deduplicate(3) + )); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + long ts1 = 500L; + long ts2 = 1500L; + long ts3 = 2500L; + + // WHEN-THEN + node.onMsg(ctxMock, TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(DEVICE_ID) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts1)))) + .build()); + then(telemetryServiceMock).should().saveTimeseries(assertArg( + actualSaveRequest -> { + assertThat(actualSaveRequest.isSaveTimeseries()).isTrue(); + assertThat(actualSaveRequest.isSaveLatest()).isTrue(); + assertThat(actualSaveRequest.isSendWsUpdate()).isTrue(); + } + )); + + clearInvocations(telemetryServiceMock); + + node.onMsg(ctxMock, TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(DEVICE_ID) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts2)))) + .build()); + then(telemetryServiceMock).should().saveTimeseries(assertArg( + actualSaveRequest -> { + assertThat(actualSaveRequest.isSaveTimeseries()).isTrue(); + assertThat(actualSaveRequest.isSaveLatest()).isFalse(); + assertThat(actualSaveRequest.isSendWsUpdate()).isFalse(); + } + )); + + clearInvocations(telemetryServiceMock); + + node.onMsg(ctxMock, TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(DEVICE_ID) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts3)))) + .build()); + then(telemetryServiceMock).should().saveTimeseries(assertArg( + actualSaveRequest -> { + assertThat(actualSaveRequest.isSaveTimeseries()).isTrue(); + assertThat(actualSaveRequest.isSaveLatest()).isTrue(); + assertThat(actualSaveRequest.isSendWsUpdate()).isFalse(); + } + )); + } + + @Test + public void givenAdvancedPersistenceSettingsWithSkipStrategiesForAllActionsAndSameMessageTwoTimes_whenOnMsg_thenSkipsSameMessageTwoTimes() throws TbNodeException { + // GIVEN + config.setPersistenceSettings(new TbMsgTimeseriesNodeConfiguration.PersistenceSettings.Advanced( + PersistenceStrategy.skip(), + PersistenceStrategy.skip(), + PersistenceStrategy.skip() + )); + + node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config))); + + var msg = TbMsg.newMsg() + .type(TbMsgType.POST_TELEMETRY_REQUEST) + .originator(DEVICE_ID) + .data(JacksonUtil.newObjectNode().put("temperature", 22.3).toString()) + .metaData(new TbMsgMetaData(Map.of("ts", "123"))) + .build(); + + // WHEN-THEN + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(never()).saveTimeseries(any()); + then(ctxMock).should(times(1)).tellSuccess(msg); + + node.onMsg(ctxMock, msg); + then(telemetryServiceMock).should(never()).saveTimeseries(any()); + then(ctxMock).should(times(2)).tellSuccess(msg); + } + + private static long extractTtlAsSeconds(TenantProfile tenantProfile) { + return TimeUnit.DAYS.toSeconds(tenantProfile.getDefaultProfileConfiguration().getDefaultStorageTtlDays()); + } + @Override protected TbNode getTestNode() { return node; From 71d43f3af2f3fbb1ce1fccd3a2424e3d3f86bd0e Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Fri, 17 Jan 2025 15:24:06 +0200 Subject: [PATCH 10/30] Save time series strategies: update configurationVersion for save time series node in rule chain JSONs --- .../data/json/edge/rule_chains/edge_root_rule_chain.json | 1 + .../json/tenant/device_profile/rule_chain_template.json | 1 + .../main/data/json/tenant/rule_chains/root_rule_chain.json | 1 + monitoring/src/main/resources/root_rule_chain.json | 6 +++--- .../src/test/resources/MqttRuleNodeTestMetadata.json | 2 +- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json b/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json index e7b4b93e98..e663ff779d 100644 --- a/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json +++ b/application/src/main/data/json/edge/rule_chains/edge_root_rule_chain.json @@ -33,6 +33,7 @@ }, "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries", + "configurationVersion": 1, "configuration": { "defaultTTL": 0, "useServerTs": false, diff --git a/application/src/main/data/json/tenant/device_profile/rule_chain_template.json b/application/src/main/data/json/tenant/device_profile/rule_chain_template.json index 37cc226a6e..325e01003f 100644 --- a/application/src/main/data/json/tenant/device_profile/rule_chain_template.json +++ b/application/src/main/data/json/tenant/device_profile/rule_chain_template.json @@ -19,6 +19,7 @@ }, "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries", + "configurationVersion": 1, "configuration": { "defaultTTL": 0, "useServerTs": false, diff --git a/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json b/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json index 7893a64fd2..4dc202d740 100644 --- a/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json +++ b/application/src/main/data/json/tenant/rule_chains/root_rule_chain.json @@ -18,6 +18,7 @@ }, "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries", + "configurationVersion": 1, "configuration": { "defaultTTL": 0, "useServerTs": false, diff --git a/monitoring/src/main/resources/root_rule_chain.json b/monitoring/src/main/resources/root_rule_chain.json index a48fd6bf35..eda3e44e1b 100644 --- a/monitoring/src/main/resources/root_rule_chain.json +++ b/monitoring/src/main/resources/root_rule_chain.json @@ -21,7 +21,7 @@ "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries", "singletonMode": false, - "configurationVersion": 0, + "configurationVersion": 1, "configuration": { "defaultTTL": 0, "useServerTs": false, @@ -277,7 +277,7 @@ "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries", "singletonMode": false, - "configurationVersion": 0, + "configurationVersion": 1, "configuration": { "defaultTTL": 0, "useServerTs": false, @@ -315,7 +315,7 @@ "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "Save Timeseries with TTL", "singletonMode": false, - "configurationVersion": 0, + "configurationVersion": 1, "configuration": { "defaultTTL": 180, "useServerTs": false, diff --git a/msa/black-box-tests/src/test/resources/MqttRuleNodeTestMetadata.json b/msa/black-box-tests/src/test/resources/MqttRuleNodeTestMetadata.json index adcb618cbe..c495e590cd 100644 --- a/msa/black-box-tests/src/test/resources/MqttRuleNodeTestMetadata.json +++ b/msa/black-box-tests/src/test/resources/MqttRuleNodeTestMetadata.json @@ -36,7 +36,7 @@ "type": "org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode", "name": "save timeseries", "singletonMode": false, - "configurationVersion": 0, + "configurationVersion": 1, "configuration": { "defaultTTL": 0, "useServerTs": false, From b898ca4d15bbb057fd61406afc4b6b2843e73bcf Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Mon, 20 Jan 2025 17:36:07 +0200 Subject: [PATCH 11/30] UI: Add time unit selector and improved save ts rule node --- .../action/timeseries-config.component.html | 60 +++--- .../action/timeseries-config.component.ts | 1 - .../common/common-rule-node-config.module.ts | 7 +- .../common/time-unit-input.component.html | 43 ++++ .../common/time-unit-input.component.ts | 184 ++++++++++++++++++ .../assets/locale/locale.constant-en_US.json | 7 +- 6 files changed, 264 insertions(+), 38 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html create mode 100644 ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html index 31e827d331..aa70914fa9 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html @@ -16,35 +16,33 @@ -->
- - rule-node-config.default-ttl - - - help - - - {{ 'rule-node-config.default-ttl-required' | translate }} - - - {{ 'rule-node-config.min-default-ttl-message' | translate }} - - -
-
- - {{ 'rule-node-config.use-server-ts' | translate }} - -
-
- - {{ 'rule-node-config.skip-latest-persistence' | translate }} - -
-
+
+ + + rule-node-config.advanced-settings + + + + + help + + +
+ + {{ 'rule-node-config.use-server-ts' | translate }} + +
+ + +
diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts index d480ec4d39..0c5283f54a 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts @@ -38,7 +38,6 @@ export class TimeseriesConfigComponent extends RuleNodeConfigurationComponent { protected onConfigurationSet(configuration: RuleNodeConfiguration) { this.timeseriesConfigForm = this.fb.group({ defaultTTL: [configuration ? configuration.defaultTTL : null, [Validators.required, Validators.min(0)]], - skipLatestPersistence: [configuration ? configuration.skipLatestPersistence : false, []], useServerTs: [configuration ? configuration.useServerTs : false, []] }); } diff --git a/ui-ngx/src/app/modules/home/components/rule-node/common/common-rule-node-config.module.ts b/ui-ngx/src/app/modules/home/components/rule-node/common/common-rule-node-config.module.ts index c72b9901c9..b6370be318 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/common/common-rule-node-config.module.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/common/common-rule-node-config.module.ts @@ -33,6 +33,7 @@ import { RelationsQueryConfigOldComponent } from './relations-query-config-old.c import { SelectAttributesComponent } from './select-attributes.component'; import { AlarmStatusSelectComponent } from './alarm-status-select.component'; import { ExampleHintComponent } from './example-hint.component'; +import { TimeUnitInputComponent } from './time-unit-input.component'; @NgModule({ declarations: [ @@ -50,7 +51,8 @@ import { ExampleHintComponent } from './example-hint.component'; RelationsQueryConfigOldComponent, SelectAttributesComponent, AlarmStatusSelectComponent, - ExampleHintComponent + ExampleHintComponent, + TimeUnitInputComponent ], imports: [ CommonModule, @@ -72,7 +74,8 @@ import { ExampleHintComponent } from './example-hint.component'; RelationsQueryConfigOldComponent, SelectAttributesComponent, AlarmStatusSelectComponent, - ExampleHintComponent + ExampleHintComponent, + TimeUnitInputComponent ] }) diff --git a/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html new file mode 100644 index 0000000000..79caab4cb7 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html @@ -0,0 +1,43 @@ + +
+ + {{ labelText }} + +
+ +
+ + {{ requiredText }} + + + {{ minErrorText }} + + + {{ maxErrorText }} + + + + rule-node-config.units + + @for (timeUnit of timeUnits; track timeUnit) { + {{ timeUnitTranslations.get(timeUnit) | translate }} + } + + +
diff --git a/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts new file mode 100644 index 0000000000..3dc0c00124 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts @@ -0,0 +1,184 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, DestroyRef, forwardRef, Input, OnInit } from '@angular/core'; +import { + AbstractControl, + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, Validators +} from '@angular/forms'; +import { TimeUnit, timeUnitTranslations } from '../rule-node-config.models'; +import { isDefinedAndNotNull, isNumeric } from '@core/utils'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { coerceBoolean, coerceNumber } from '@shared/decorators/coercion'; +import { DAY, HOUR, MINUTE, SECOND } from '@shared/models/time/time.models'; + +interface TimeUnitInputModel { + time: number; + timeUnit: TimeUnit +} + +@Component({ + selector: 'tb-time-unit-input', + templateUrl: './time-unit-input.component.html', + providers: [{ + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => TimeUnitInputComponent), + multi: true + },{ + provide: NG_VALIDATORS, + useExisting: forwardRef(() => TimeUnitInputComponent), + multi: true + }] +}) +export class TimeUnitInputComponent implements ControlValueAccessor, Validator, OnInit { + + @Input() + labelText: string; + + @Input() + @coerceBoolean() + required: boolean; + + @Input() + requiredText: string; + + @Input() + minErrorText: string; + + @Input() + @coerceNumber() + maxTime: number; + + @Input() + maxErrorText: string; + + timeUnits = Object.values(TimeUnit).filter(item => item !== TimeUnit.MILLISECONDS) as TimeUnit[]; + + timeUnitTranslations = timeUnitTranslations; + + timeInputForm = this.fb.group({ + time: [0, Validators.min(0)], + timeUnit: [TimeUnit.SECONDS] + }); + + private timeIntervalsInSec = new Map([ + [TimeUnit.DAYS, DAY/SECOND], + [TimeUnit.HOURS, HOUR/SECOND], + [TimeUnit.MINUTES, MINUTE/SECOND], + [TimeUnit.SECONDS, SECOND/SECOND], + ]); + + private modelValue: number; + + private propagateChange: (value: any) => void = () => {}; + + constructor(private fb: FormBuilder, + private destroyRef: DestroyRef) { + } + + ngOnInit() { + if(this.required || this.maxTime) { + const timeControl = this.timeInputForm.get('time'); + const validators = []; + if (this.required) { + validators.push(Validators.required); + } + if (this.maxTime) { + validators.push((control: AbstractControl) => + Validators.max(Math.floor(this.maxTime / this.timeIntervalsInSec.get(this.timeInputForm.get('timeUnit').value)))(control) + ); + } + + timeControl.setValidators(validators); + timeControl.updateValueAndValidity({ emitEvent: false }); + } + + this.timeInputForm.get('timeUnit').valueChanges.pipe( + takeUntilDestroyed(this.destroyRef) + ).subscribe(() => { + this.timeInputForm.get('time').updateValueAndValidity({onlySelf: true}); + this.timeInputForm.get('time').markAsTouched({onlySelf: true}); + }); + + this.timeInputForm.valueChanges.pipe( + takeUntilDestroyed(this.destroyRef) + ).subscribe(value => { + this.updatedModel(value); + }); + } + + registerOnChange(fn: any) { + this.propagateChange = fn; + } + + registerOnTouched(_fn: any) { + } + + setDisabledState(isDisabled: boolean) { + if (isDisabled) { + this.timeInputForm.disable({emitEvent: false}); + } else { + this.timeInputForm.enable({emitEvent: false}); + } + } + + writeValue(sec: number) { + if (sec !== this.modelValue) { + if (isDefinedAndNotNull(sec) && isNumeric(sec) && Number(sec) !== 0) { + this.timeInputForm.patchValue(this.parseTime(sec), {emitEvent: false}); + this.modelValue = sec; + } else { + this.timeInputForm.patchValue({ + time: 0, + timeUnit: TimeUnit.SECONDS + }, {emitEvent: false}); + this.modelValue = 0; + } + } + } + + validate(): ValidationErrors | null { + return this.timeInputForm.valid ? null : { + timeInput: false + }; + } + + private updatedModel(value: Partial) { + const time = value.time * this.timeIntervalsInSec.get(value.timeUnit); + if (this.modelValue !== time) { + this.modelValue = time; + this.propagateChange(time); + } + } + + private parseTime(value: number): TimeUnitInputModel { + for (const [timeUnit, timeValue] of this.timeIntervalsInSec) { + const calc = value / timeValue; + if (Number.isInteger(calc)) { + return { + time: calc, + timeUnit: timeUnit + } + } + } + } + +} diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 2fb67adbba..b6aa720a53 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -4541,7 +4541,7 @@ "originator-entity": "Entity by name pattern", "clone-message": "Clone message", "transform": "Transform", - "default-ttl": "Default TTL in seconds", + "default-ttl": "Default TTL", "default-ttl-required": "Default TTL is required.", "default-ttl-hint": "Rule node will fetch Time-to-Live (TTL) value from the message metadata. If no value is present, it defaults to the TTL specified in the configuration. If the value is set to 0, the TTL from the tenant profile configuration will be applied.", "default-ttl-zero-hint": "TTL will not be applied if its value is set to 0.", @@ -4906,9 +4906,7 @@ "general-pattern-hint": "Use ${metadataKey} for value from metadata, $[messageKey] for value from message body.", "alarm-severity-pattern-hint": "Use ${metadataKey} for value from metadata, $[messageKey] for value from message body. Alarm severity should be system (CRITICAL, MAJOR etc.)", "output-node-name-hint": "The rule node name corresponds to the relation type of the output message, and it is used to forward messages to other rule nodes in the caller rule chain.", - "skip-latest-persistence": "Skip latest persistence", - "skip-latest-persistence-hint": "Rule node will not update values for incoming keys for the latest time series data. Useful for highly loaded use-cases to decrease the pressure on the DB.", - "use-server-ts": "Use server ts", + "use-server-ts": "Use server timestamp", "use-server-ts-hint": "Rule node will use the timestamp of message processing instead of the timestamp from the message. Useful for all sorts of sequential processing if you merge messages from multiple sources (devices, assets, etc).", "kv-map-pattern-hint": "All input fields support templatization. Use $[messageKey] to extract value from the message and ${metadataKey} to extract value from the metadata.", "kv-map-single-pattern-hint": "Input field support templatization. Use $[messageKey] to extract value from the message and ${metadataKey} to extract value from the metadata.", @@ -5067,6 +5065,7 @@ "request-timeout-required": "Request timeout is required", "request-timeout-min": "Min request timeout is 0", "request-timeout-hint": "The amount of time to wait in seconds for the request to complete before giving up and timing out. A value of 0 means infinity, and is not recommended.", + "units": "Units", "tell-failure-aws-lambda": "Tell Failure if AWS Lambda function execution raises exception", "tell-failure-aws-lambda-hint": "Rule node forces failure of message processing if AWS Lambda function execution raises exception.", "key-val": { From 996b8997fdb6df1f2f3109a585b4866d0ca25be8 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Tue, 21 Jan 2025 11:14:22 +0200 Subject: [PATCH 12/30] Save time series strategies: simplify SQL upgrade script --- .../main/data/upgrade/basic/schema_update.sql | 78 +++++++------------ 1 file changed, 27 insertions(+), 51 deletions(-) diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index 6aaf8c3dbd..31833112ad 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -221,60 +221,36 @@ DO $$ WHERE table_name = 'rule_node' ) THEN - -- CREATE JSON validation function - CREATE OR REPLACE FUNCTION is_valid_jsonb(input text) - RETURNS boolean - LANGUAGE plpgsql - AS $func$ - DECLARE - dummy JSONB; - BEGIN - dummy := input::jsonb; - RETURN true; - EXCEPTION - WHEN others THEN - RETURN false; - END; - $func$; - UPDATE rule_node - SET configuration = CASE - -- Case 1: If configuration is NULL, invalid JSON, or not a JSON object - set default configuration - WHEN configuration IS NULL - OR NOT is_valid_jsonb(configuration) - OR jsonb_typeof(configuration::jsonb) <> 'object' - THEN jsonb_build_object( - 'defaultTTL', 0, - 'useServerTs', false, - 'persistenceSettings', jsonb_build_object('type', 'ON_EVERY_MESSAGE') - ) - -- Case 2: If a valid JSON object with persistenceSettings (rule node was already upgraded) - leave unchanged - WHEN configuration::jsonb ? 'persistenceSettings' - THEN configuration::jsonb - -- Case 3: If a valid JSON object without persistenceSettings and skipLatestPersistence = 'true' (string 'true' or boolean true) - set latest to SKIP - WHEN configuration::jsonb ->> 'skipLatestPersistence' = 'true' - THEN (configuration::jsonb - 'skipLatestPersistence') - || jsonb_build_object( - 'persistenceSettings', jsonb_build_object( - 'type', 'ADVANCED', - 'timeseries', jsonb_build_object('type', 'ON_EVERY_MESSAGE'), - 'latest', jsonb_build_object('type', 'SKIP'), - 'webSockets', jsonb_build_object('type', 'ON_EVERY_MESSAGE') - ) - ) - -- Case 4: If a valid JSON object without persistenceSettings and skipLatestPersistence not 'true' (everything else) - set all to ON_EVERY_MESSAGE - ELSE (configuration::jsonb - 'skipLatestPersistence') - || jsonb_build_object( - 'persistenceSettings', jsonb_build_object( - 'type', 'ON_EVERY_MESSAGE' - ) - ) - END::text, + SET configuration = ( + (configuration::jsonb - 'skipLatestPersistence') + || jsonb_build_object( + 'persistenceSettings', jsonb_build_object( + 'type', 'ADVANCED', + 'timeseries', jsonb_build_object('type', 'ON_EVERY_MESSAGE'), + 'latest', jsonb_build_object('type', 'SKIP'), + 'webSockets', jsonb_build_object('type', 'ON_EVERY_MESSAGE') + ) + ) + )::text, configuration_version = 1 - WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode' AND configuration_version = 0; + WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode' + AND configuration_version = 0 + AND configuration::jsonb ->> 'skipLatestPersistence' = 'true'; - -- Drop the helper function - DROP FUNCTION is_valid_jsonb(text); + UPDATE rule_node + SET configuration = ( + (configuration::jsonb - 'skipLatestPersistence') + || jsonb_build_object( + 'persistenceSettings', jsonb_build_object( + 'type', 'ON_EVERY_MESSAGE' + ) + ) + )::text, + configuration_version = 1 + WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgTimeseriesNode' + AND configuration_version = 0 + AND (configuration::jsonb ->> 'skipLatestPersistence' != 'true' OR configuration::jsonb ->> 'skipLatestPersistence' IS NULL); END IF; END; From f419a6e438382e79eb759fb4a6a0b9ef7a14fdc6 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Tue, 21 Jan 2025 11:17:05 +0200 Subject: [PATCH 13/30] Save time series strategies: remove unnecessary check in Java upgrade script --- .../thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 422065f81f..7a32ee47c1 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -193,9 +193,6 @@ public class TbMsgTimeseriesNode implements TbNode { boolean hasChanges = false; switch (fromVersion) { case 0: - if (oldConfiguration.has("persistenceSettings") && !oldConfiguration.has("skipLatestPersistence")) { - break; - } hasChanges = true; JsonNode skipLatestPersistence = oldConfiguration.get("skipLatestPersistence"); if (skipLatestPersistence != null && "true".equals(skipLatestPersistence.asText())) { From 39e47cd484c7e0afb6d642b88c3f388728c037f2 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Tue, 21 Jan 2025 14:26:08 +0200 Subject: [PATCH 14/30] Save time series strategies: refactor boolean flags in TimeseriesSaveRequest.java to SaveActions nested record --- .../DefaultTbEntityViewService.java | 4 +- .../DefaultTelemetrySubscriptionService.java | 19 ++++--- .../DefaultTbEntityViewServiceTest.java | 4 +- ...faultTelemetrySubscriptionServiceTest.java | 28 +++------- .../engine/api/TimeseriesSaveRequest.java | 33 +++++------- .../engine/api/TimeseriesSaveRequestTest.java | 26 +++++++-- .../engine/telemetry/TbMsgTimeseriesNode.java | 54 +++++++------------ .../rule/engine/math/TbMathNodeTest.java | 4 +- .../telemetry/TbMsgTimeseriesNodeTest.java | 42 ++++----------- 9 files changed, 86 insertions(+), 128 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java index ee807da750..3ff1387103 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java @@ -348,9 +348,7 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen .tenantId(entityView.getTenantId()) .entityId(entityId) .entries(latestValues) - .saveTimeseries(false) - .saveLatest(true) - .sendWsUpdate(true) + .saveActions(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS) .callback(new FutureCallback() { @Override public void onSuccess(@Nullable Void tmp) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index dbd7967989..ab95ec6cba 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -118,10 +118,10 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer EntityId entityId = request.getEntityId(); checkInternalEntity(entityId); boolean sysTenant = TenantId.SYS_TENANT_ID.equals(tenantId) || tenantId == null; - if (sysTenant || !request.isSaveTimeseries() || apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) { + if (sysTenant || !request.getSaveActions().saveTimeseries() || apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) { KvUtils.validate(request.getEntries(), valueNoXssValidation); ListenableFuture future = saveTimeseriesInternal(request); - if (request.isSaveTimeseries()) { + if (request.getSaveActions().saveTimeseries()) { FutureCallback callback = getApiUsageCallback(tenantId, request.getCustomerId(), sysTenant, request.getCallback()); Futures.addCallback(future, callback, tsCallBackExecutor); } @@ -134,22 +134,23 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer public ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); + TimeseriesSaveRequest.SaveActions saveActions = request.getSaveActions(); ListenableFuture saveFuture; - if (request.isSaveTimeseries() && request.isSaveLatest()) { + if (saveActions.saveTimeseries() && saveActions.saveLatest()) { saveFuture = tsService.save(tenantId, entityId, request.getEntries(), request.getTtl()); - } else if (request.isSaveLatest()) { + } else if (saveActions.saveLatest()) { saveFuture = Futures.transform(tsService.saveLatest(tenantId, entityId, request.getEntries()), result -> 0, MoreExecutors.directExecutor()); - } else if (request.isSaveTimeseries()) { + } else if (saveActions.saveTimeseries()) { saveFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } else { saveFuture = Futures.immediateFuture(0); } addMainCallback(saveFuture, request.getCallback()); - if (request.isSendWsUpdate()) { + if (saveActions.sendWsUpdate()) { addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); } - if (request.isSaveLatest()) { + if (saveActions.saveLatest()) { copyLatestToEntityViews(tenantId, entityId, request.getEntries()); } return saveFuture; @@ -236,9 +237,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer .tenantId(tenantId) .entityId(entityView.getId()) .entries(entityViewLatest) - .saveTimeseries(false) - .saveLatest(true) - .sendWsUpdate(true) + .saveActions(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS) .callback(new FutureCallback<>() { @Override public void onSuccess(@Nullable Void tmp) {} diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java index b36890f775..d6f803c1fe 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java @@ -93,9 +93,7 @@ class DefaultTbEntityViewServiceTest { .entityId(entityView.getId()) .entries(latest) .ttl(0L) - .saveTimeseries(false) - .saveLatest(true) - .sendWsUpdate(true) + .saveActions(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS) .build(); var actualCopyLatestRequest = captor.getValue(); diff --git a/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java index 8162ef4555..21d3aac607 100644 --- a/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java @@ -173,9 +173,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveTimeseries(true) - .saveLatest(false) - .sendWsUpdate(false) + .saveActions(new TimeseriesSaveRequest.SaveActions(true, false, false)) .callback(emptyCallback) .build(); @@ -195,9 +193,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveTimeseries(false) - .saveLatest(true) - .sendWsUpdate(true) + .saveActions(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS) .callback(emptyCallback) .build(); @@ -220,9 +216,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveTimeseries(true) - .saveLatest(true) - .sendWsUpdate(true) + .saveActions(TimeseriesSaveRequest.SaveActions.SAVE_ALL) .future(future) .build(); @@ -248,9 +242,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveTimeseries(false) - .saveLatest(true) - .sendWsUpdate(true) + .saveActions(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS) .future(future) .build(); @@ -283,9 +275,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveTimeseries(false) - .saveLatest(true) - .sendWsUpdate(false) + .saveActions(new TimeseriesSaveRequest.SaveActions(false, true, false)) .callback(emptyCallback) .build(); @@ -312,9 +302,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveTimeseries(true) - .saveLatest(false) - .sendWsUpdate(false) + .saveActions(new TimeseriesSaveRequest.SaveActions(true, false, false)) .callback(emptyCallback) .build(); @@ -340,9 +328,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveTimeseries(saveTimeseries) - .saveLatest(saveLatest) - .sendWsUpdate(sendWsUpdate) + .saveActions(new TimeseriesSaveRequest.SaveActions(saveTimeseries, saveLatest, sendWsUpdate)) .callback(emptyCallback) .build(); diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java index 1ad0fd3408..d1122cc70d 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java @@ -38,11 +38,18 @@ public class TimeseriesSaveRequest { private final EntityId entityId; private final List entries; private final long ttl; - private final boolean saveTimeseries; - private final boolean saveLatest; - private final boolean sendWsUpdate; + private final SaveActions saveActions; private final FutureCallback callback; + public record SaveActions(boolean saveTimeseries, boolean saveLatest, boolean sendWsUpdate) { + + public static final SaveActions SAVE_ALL = new SaveActions(true, true, true); + public static final SaveActions WS_ONLY = new SaveActions(false, false, true); + public static final SaveActions LATEST_AND_WS = new SaveActions(false, true, true); + public static final SaveActions SKIP_ALL = new SaveActions(false, false, false); + + } + public static Builder builder() { return new Builder(); } @@ -54,9 +61,7 @@ public class TimeseriesSaveRequest { private EntityId entityId; private List entries; private long ttl; - private boolean saveTimeseries = true; - private boolean saveLatest = true; - private boolean sendWsUpdate = true; + private SaveActions saveActions = SaveActions.SAVE_ALL; private FutureCallback callback; Builder() {} @@ -94,18 +99,8 @@ public class TimeseriesSaveRequest { return this; } - public Builder saveTimeseries(boolean saveTimeseries) { - this.saveTimeseries = saveTimeseries; - return this; - } - - public Builder saveLatest(boolean saveLatest) { - this.saveLatest = saveLatest; - return this; - } - - public Builder sendWsUpdate(boolean sendWsUpdate) { - this.sendWsUpdate = sendWsUpdate; + public Builder saveActions(SaveActions settings) { + this.saveActions = settings; return this; } @@ -129,7 +124,7 @@ public class TimeseriesSaveRequest { } public TimeseriesSaveRequest build() { - return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveTimeseries, saveLatest, sendWsUpdate, callback); + return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveActions, callback); } } diff --git a/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java b/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java index cf05f9905b..38ce10a272 100644 --- a/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java +++ b/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java @@ -22,12 +22,30 @@ import static org.assertj.core.api.Assertions.assertThat; class TimeseriesSaveRequestTest { @Test - void testBooleanFlagsDefaultToTrue() { + void testDefaultSaveActionsAreSaveAll() { var request = TimeseriesSaveRequest.builder().build(); - assertThat(request.isSaveTimeseries()).isTrue(); - assertThat(request.isSaveLatest()).isTrue(); - assertThat(request.isSendWsUpdate()).isTrue(); + assertThat(request.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL); + } + + @Test + void testSaveActionsSaveAll() { + assertThat(TimeseriesSaveRequest.SaveActions.SAVE_ALL).isEqualTo(new TimeseriesSaveRequest.SaveActions(true, true, true)); + } + + @Test + void testSaveActionsWsOnly() { + assertThat(TimeseriesSaveRequest.SaveActions.WS_ONLY).isEqualTo(new TimeseriesSaveRequest.SaveActions(false, false, true)); + } + + @Test + void testSaveActionsLatestAndWs() { + assertThat(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS).isEqualTo(new TimeseriesSaveRequest.SaveActions(false, true, true)); + } + + @Test + void testSaveActionsSkipAll() { + assertThat(TimeseriesSaveRequest.SaveActions.SKIP_ALL).isEqualTo(new TimeseriesSaveRequest.SaveActions(false, false, false)); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 7a32ee47c1..658d635fae 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -107,13 +107,10 @@ public class TbMsgTimeseriesNode implements TbNode { } long ts = computeTs(msg, config.isUseServerTs()); - PersistenceDecision persistenceDecision = makePersistenceDecision(ts, msg.getOriginator().getId()); - boolean saveTimeseries = persistenceDecision.saveTimeseries(); - boolean saveLatest = persistenceDecision.saveLatest(); - boolean sendWsUpdate = persistenceDecision.sendWsUpdate(); + TimeseriesSaveRequest.SaveActions saveActions = determineSaveActions(ts, msg.getOriginator().getId()); // short-circuit - if (!saveTimeseries && !saveLatest && !sendWsUpdate) { + if (!saveActions.saveTimeseries() && !saveActions.saveLatest() && !saveActions.sendWsUpdate()) { ctx.tellSuccess(msg); return; } @@ -141,9 +138,7 @@ public class TbMsgTimeseriesNode implements TbNode { .entityId(msg.getOriginator()) .entries(tsKvEntryList) .ttl(ttl) - .saveTimeseries(saveTimeseries) - .saveLatest(saveLatest) - .sendWsUpdate(sendWsUpdate) + .saveActions(saveActions) .callback(new TelemetryNodeCallback(ctx, msg)) .build()); } @@ -152,35 +147,26 @@ public class TbMsgTimeseriesNode implements TbNode { return ignoreMetadataTs ? System.currentTimeMillis() : msg.getMetaDataTs(); } - private record PersistenceDecision(boolean saveTimeseries, boolean saveLatest, boolean sendWsUpdate) {} - - private PersistenceDecision makePersistenceDecision(long ts, UUID originatorUuid) { - boolean saveTimeseries; - boolean saveLatest; - boolean sendWsUpdate; - + private TimeseriesSaveRequest.SaveActions determineSaveActions(long ts, UUID originatorUuid) { if (persistenceSettings instanceof OnEveryMessage) { - saveTimeseries = true; - saveLatest = true; - sendWsUpdate = true; - } else if (persistenceSettings instanceof WebSocketsOnly) { - saveTimeseries = false; - saveLatest = false; - sendWsUpdate = true; - } else if (persistenceSettings instanceof Deduplicate deduplicate) { + return TimeseriesSaveRequest.SaveActions.SAVE_ALL; + } + if (persistenceSettings instanceof WebSocketsOnly) { + return TimeseriesSaveRequest.SaveActions.WS_ONLY; + } + if (persistenceSettings instanceof Deduplicate deduplicate) { boolean isFirstMsgInInterval = deduplicate.getDeduplicateStrategy().shouldPersist(ts, originatorUuid); - saveTimeseries = isFirstMsgInInterval; - saveLatest = isFirstMsgInInterval; - sendWsUpdate = isFirstMsgInInterval; - } else if (persistenceSettings instanceof Advanced advanced) { - saveTimeseries = advanced.timeseries().shouldPersist(ts, originatorUuid); - saveLatest = advanced.latest().shouldPersist(ts, originatorUuid); - sendWsUpdate = advanced.webSockets().shouldPersist(ts, originatorUuid); - } else { // should not happen - throw new IllegalArgumentException("Unknown persistence settings type: " + persistenceSettings.getClass().getSimpleName()); + return isFirstMsgInInterval ? TimeseriesSaveRequest.SaveActions.SAVE_ALL : TimeseriesSaveRequest.SaveActions.SKIP_ALL; } - - return new PersistenceDecision(saveTimeseries, saveLatest, sendWsUpdate); + if (persistenceSettings instanceof Advanced advanced) { + return new TimeseriesSaveRequest.SaveActions( + advanced.timeseries().shouldPersist(ts, originatorUuid), + advanced.latest().shouldPersist(ts, originatorUuid), + advanced.webSockets().shouldPersist(ts, originatorUuid) + ); + } + // should not happen + throw new IllegalArgumentException("Unknown persistence settings type: " + persistenceSettings.getClass().getSimpleName()); } @Override diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java index 360fbce5b0..1270c8cf6c 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java @@ -533,7 +533,7 @@ public class TbMathNodeTest { verify(ctx, timeout(TIMEOUT)).tellSuccess(msgCaptor.capture()); verify(telemetryService, times(1)).saveTimeseries(assertArg(request -> { assertThat(request.getEntries()).size().isOne(); - assertThat(request.isSaveLatest()).isTrue(); + assertThat(request.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL); })); TbMsg resultMsg = msgCaptor.getValue(); @@ -569,7 +569,7 @@ public class TbMathNodeTest { verify(ctx, timeout(TIMEOUT)).tellSuccess(msgCaptor.capture()); verify(telemetryService, times(1)).saveTimeseries(assertArg(request -> { assertThat(request.getEntries()).size().isOne(); - assertThat(request.isSaveLatest()).isTrue(); + assertThat(request.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL); })); TbMsg resultMsg = msgCaptor.getValue(); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java index d0727b1a3c..15904ca1ff 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java @@ -207,7 +207,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { assertThat(request.getEntityId()).isEqualTo(DEVICE_ID); assertThat(request.getEntries()).usingRecursiveFieldByFieldElementComparatorIgnoringFields("ts").containsExactlyElementsOf(expectedList); assertThat(request.getTtl()).isEqualTo(extractTtlAsSeconds(tenantProfile)); - assertThat(request.isSaveLatest()).isTrue(); + assertThat(request.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL); assertThat(request.getCallback()).isInstanceOf(TelemetryNodeCallback.class); })); verify(ctxMock).tellSuccess(msg); @@ -264,9 +264,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { assertThat(request.getEntityId()).isEqualTo(DEVICE_ID); assertThat(request.getEntries()).containsExactlyElementsOf(expectedList); assertThat(request.getTtl()).isEqualTo(config.getDefaultTTL()); - assertThat(request.isSaveTimeseries()).isTrue(); - assertThat(request.isSaveLatest()).isFalse(); - assertThat(request.isSendWsUpdate()).isTrue(); + assertThat(request.getSaveActions()).isEqualTo(new TimeseriesSaveRequest.SaveActions(true, false, true)); assertThat(request.getCallback()).isInstanceOf(TelemetryNodeCallback.class); })); verify(ctxMock).tellSuccess(msg); @@ -305,7 +303,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { assertThat(request.getCustomerId()).isNull(); assertThat(request.getEntityId()).isEqualTo(DEVICE_ID); assertThat(request.getTtl()).isEqualTo(expectedTtl); - assertThat(request.isSaveLatest()).isTrue(); + assertThat(request.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL); assertThat(request.getCallback()).isInstanceOf(TelemetryNodeCallback.class); })); } @@ -354,9 +352,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entityId(msg.getOriginator()) .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) - .saveTimeseries(true) - .saveLatest(true) - .sendWsUpdate(true) + .saveActions(TimeseriesSaveRequest.SaveActions.SAVE_ALL) .build(); node.onMsg(ctxMock, msg); @@ -391,9 +387,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entityId(msg.getOriginator()) .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) - .saveTimeseries(true) - .saveLatest(true) - .sendWsUpdate(true) + .saveActions(TimeseriesSaveRequest.SaveActions.SAVE_ALL) .build(); node.onMsg(ctxMock, msg); @@ -428,9 +422,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entityId(msg.getOriginator()) .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) - .saveTimeseries(false) - .saveLatest(false) - .sendWsUpdate(true) + .saveActions(TimeseriesSaveRequest.SaveActions.WS_ONLY) .build(); node.onMsg(ctxMock, msg); @@ -469,9 +461,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entityId(msg.getOriginator()) .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) - .saveTimeseries(true) - .saveLatest(true) - .sendWsUpdate(true) + .saveActions(TimeseriesSaveRequest.SaveActions.SAVE_ALL) .build(); node.onMsg(ctxMock, msg); @@ -508,11 +498,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts1)))) .build()); then(telemetryServiceMock).should().saveTimeseries(assertArg( - actualSaveRequest -> { - assertThat(actualSaveRequest.isSaveTimeseries()).isTrue(); - assertThat(actualSaveRequest.isSaveLatest()).isTrue(); - assertThat(actualSaveRequest.isSendWsUpdate()).isTrue(); - } + actualSaveRequest -> assertThat(actualSaveRequest.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL) )); clearInvocations(telemetryServiceMock); @@ -524,11 +510,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts2)))) .build()); then(telemetryServiceMock).should().saveTimeseries(assertArg( - actualSaveRequest -> { - assertThat(actualSaveRequest.isSaveTimeseries()).isTrue(); - assertThat(actualSaveRequest.isSaveLatest()).isFalse(); - assertThat(actualSaveRequest.isSendWsUpdate()).isFalse(); - } + actualSaveRequest -> assertThat(actualSaveRequest.getSaveActions()).isEqualTo(new TimeseriesSaveRequest.SaveActions(true, false, false)) )); clearInvocations(telemetryServiceMock); @@ -540,11 +522,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts3)))) .build()); then(telemetryServiceMock).should().saveTimeseries(assertArg( - actualSaveRequest -> { - assertThat(actualSaveRequest.isSaveTimeseries()).isTrue(); - assertThat(actualSaveRequest.isSaveLatest()).isTrue(); - assertThat(actualSaveRequest.isSendWsUpdate()).isFalse(); - } + actualSaveRequest -> assertThat(actualSaveRequest.getSaveActions()).isEqualTo(new TimeseriesSaveRequest.SaveActions(true, true, false)) )); } From 148521eddf6bdd95b44ab0986f83f7b8895656d3 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Tue, 21 Jan 2025 16:10:35 +0200 Subject: [PATCH 15/30] Save time series strategies: use @NotNull annotation instead of manual check --- .../rule/engine/telemetry/TbMsgTimeseriesNode.java | 3 --- .../telemetry/TbMsgTimeseriesNodeConfiguration.java | 2 ++ .../engine/telemetry/TbMsgTimeseriesNodeTest.java | 11 ++++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 658d635fae..eb2862a0da 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -89,9 +89,6 @@ public class TbMsgTimeseriesNode implements TbNode { ctx.addTenantProfileListener(this::onTenantProfileUpdate); onTenantProfileUpdate(ctx.getTenantProfile()); persistenceSettings = config.getPersistenceSettings(); - if (persistenceSettings == null) { - throw new TbNodeException("Persistence settings cannot be null", true); - } } private void onTenantProfileUpdate(TenantProfile tenantProfile) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java index f702ecc6da..fe22e42710 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import jakarta.validation.constraints.NotNull; import lombok.Data; import lombok.Getter; import org.thingsboard.rule.engine.api.NodeConfiguration; @@ -37,6 +38,7 @@ public class TbMsgTimeseriesNodeConfiguration implements NodeConfiguration node.init(ctxMock, new TbNodeConfiguration(JacksonUtil.valueToTree(config)))) - .isInstanceOf(TbNodeException.class) - .matches(e -> ((TbNodeException) e).isUnrecoverable()) - .hasMessage("Persistence settings cannot be null"); + assertThatThrownBy(() -> ConstraintValidator.validateFields(config)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Validation error: persistenceSettings must not be null"); } @ParameterizedTest From e009967fa74e58a0da5fd883d4a452f0303fba95 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Tue, 21 Jan 2025 16:21:28 +0200 Subject: [PATCH 16/30] Save time series strategies: rename SaveActions to Strategy --- .../DefaultTbEntityViewService.java | 2 +- .../DefaultTelemetrySubscriptionService.java | 18 ++++++++--------- .../DefaultTbEntityViewServiceTest.java | 2 +- ...faultTelemetrySubscriptionServiceTest.java | 14 ++++++------- .../engine/api/TimeseriesSaveRequest.java | 20 +++++++++---------- .../engine/api/TimeseriesSaveRequestTest.java | 20 +++++++++---------- .../engine/telemetry/TbMsgTimeseriesNode.java | 16 +++++++-------- .../rule/engine/math/TbMathNodeTest.java | 4 ++-- .../telemetry/TbMsgTimeseriesNodeTest.java | 20 +++++++++---------- 9 files changed, 58 insertions(+), 58 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java index 3ff1387103..2c384aa493 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java @@ -348,7 +348,7 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen .tenantId(entityView.getTenantId()) .entityId(entityId) .entries(latestValues) - .saveActions(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS) + .strategy(TimeseriesSaveRequest.Strategy.LATEST_AND_WS) .callback(new FutureCallback() { @Override public void onSuccess(@Nullable Void tmp) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index ab95ec6cba..cab2f18dc7 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -118,10 +118,10 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer EntityId entityId = request.getEntityId(); checkInternalEntity(entityId); boolean sysTenant = TenantId.SYS_TENANT_ID.equals(tenantId) || tenantId == null; - if (sysTenant || !request.getSaveActions().saveTimeseries() || apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) { + if (sysTenant || !request.getStrategy().saveTimeseries() || apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) { KvUtils.validate(request.getEntries(), valueNoXssValidation); ListenableFuture future = saveTimeseriesInternal(request); - if (request.getSaveActions().saveTimeseries()) { + if (request.getStrategy().saveTimeseries()) { FutureCallback callback = getApiUsageCallback(tenantId, request.getCustomerId(), sysTenant, request.getCallback()); Futures.addCallback(future, callback, tsCallBackExecutor); } @@ -134,23 +134,23 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer public ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); - TimeseriesSaveRequest.SaveActions saveActions = request.getSaveActions(); + TimeseriesSaveRequest.Strategy strategy = request.getStrategy(); ListenableFuture saveFuture; - if (saveActions.saveTimeseries() && saveActions.saveLatest()) { + if (strategy.saveTimeseries() && strategy.saveLatest()) { saveFuture = tsService.save(tenantId, entityId, request.getEntries(), request.getTtl()); - } else if (saveActions.saveLatest()) { + } else if (strategy.saveLatest()) { saveFuture = Futures.transform(tsService.saveLatest(tenantId, entityId, request.getEntries()), result -> 0, MoreExecutors.directExecutor()); - } else if (saveActions.saveTimeseries()) { + } else if (strategy.saveTimeseries()) { saveFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } else { saveFuture = Futures.immediateFuture(0); } addMainCallback(saveFuture, request.getCallback()); - if (saveActions.sendWsUpdate()) { + if (strategy.sendWsUpdate()) { addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); } - if (saveActions.saveLatest()) { + if (strategy.saveLatest()) { copyLatestToEntityViews(tenantId, entityId, request.getEntries()); } return saveFuture; @@ -237,7 +237,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer .tenantId(tenantId) .entityId(entityView.getId()) .entries(entityViewLatest) - .saveActions(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS) + .strategy(TimeseriesSaveRequest.Strategy.LATEST_AND_WS) .callback(new FutureCallback<>() { @Override public void onSuccess(@Nullable Void tmp) {} diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java index d6f803c1fe..aa6bfde935 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewServiceTest.java @@ -93,7 +93,7 @@ class DefaultTbEntityViewServiceTest { .entityId(entityView.getId()) .entries(latest) .ttl(0L) - .saveActions(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS) + .strategy(TimeseriesSaveRequest.Strategy.LATEST_AND_WS) .build(); var actualCopyLatestRequest = captor.getValue(); diff --git a/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java index 21d3aac607..10fdd85504 100644 --- a/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionServiceTest.java @@ -173,7 +173,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveActions(new TimeseriesSaveRequest.SaveActions(true, false, false)) + .strategy(new TimeseriesSaveRequest.Strategy(true, false, false)) .callback(emptyCallback) .build(); @@ -193,7 +193,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveActions(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS) + .strategy(TimeseriesSaveRequest.Strategy.LATEST_AND_WS) .callback(emptyCallback) .build(); @@ -216,7 +216,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveActions(TimeseriesSaveRequest.SaveActions.SAVE_ALL) + .strategy(TimeseriesSaveRequest.Strategy.SAVE_ALL) .future(future) .build(); @@ -242,7 +242,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveActions(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS) + .strategy(TimeseriesSaveRequest.Strategy.LATEST_AND_WS) .future(future) .build(); @@ -275,7 +275,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveActions(new TimeseriesSaveRequest.SaveActions(false, true, false)) + .strategy(new TimeseriesSaveRequest.Strategy(false, true, false)) .callback(emptyCallback) .build(); @@ -302,7 +302,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveActions(new TimeseriesSaveRequest.SaveActions(true, false, false)) + .strategy(new TimeseriesSaveRequest.Strategy(true, false, false)) .callback(emptyCallback) .build(); @@ -328,7 +328,7 @@ class DefaultTelemetrySubscriptionServiceTest { .entityId(entityId) .entries(sampleTelemetry) .ttl(sampleTtl) - .saveActions(new TimeseriesSaveRequest.SaveActions(saveTimeseries, saveLatest, sendWsUpdate)) + .strategy(new TimeseriesSaveRequest.Strategy(saveTimeseries, saveLatest, sendWsUpdate)) .callback(emptyCallback) .build(); diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java index d1122cc70d..fb667fbfb2 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java @@ -38,15 +38,15 @@ public class TimeseriesSaveRequest { private final EntityId entityId; private final List entries; private final long ttl; - private final SaveActions saveActions; + private final Strategy strategy; private final FutureCallback callback; - public record SaveActions(boolean saveTimeseries, boolean saveLatest, boolean sendWsUpdate) { + public record Strategy(boolean saveTimeseries, boolean saveLatest, boolean sendWsUpdate) { - public static final SaveActions SAVE_ALL = new SaveActions(true, true, true); - public static final SaveActions WS_ONLY = new SaveActions(false, false, true); - public static final SaveActions LATEST_AND_WS = new SaveActions(false, true, true); - public static final SaveActions SKIP_ALL = new SaveActions(false, false, false); + public static final Strategy SAVE_ALL = new Strategy(true, true, true); + public static final Strategy WS_ONLY = new Strategy(false, false, true); + public static final Strategy LATEST_AND_WS = new Strategy(false, true, true); + public static final Strategy SKIP_ALL = new Strategy(false, false, false); } @@ -61,7 +61,7 @@ public class TimeseriesSaveRequest { private EntityId entityId; private List entries; private long ttl; - private SaveActions saveActions = SaveActions.SAVE_ALL; + private Strategy strategy = Strategy.SAVE_ALL; private FutureCallback callback; Builder() {} @@ -99,8 +99,8 @@ public class TimeseriesSaveRequest { return this; } - public Builder saveActions(SaveActions settings) { - this.saveActions = settings; + public Builder strategy(Strategy strategy) { + this.strategy = strategy; return this; } @@ -124,7 +124,7 @@ public class TimeseriesSaveRequest { } public TimeseriesSaveRequest build() { - return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveActions, callback); + return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, strategy, callback); } } diff --git a/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java b/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java index 38ce10a272..321892991e 100644 --- a/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java +++ b/rule-engine/rule-engine-api/src/test/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequestTest.java @@ -22,30 +22,30 @@ import static org.assertj.core.api.Assertions.assertThat; class TimeseriesSaveRequestTest { @Test - void testDefaultSaveActionsAreSaveAll() { + void testDefaultSaveStrategyIsSaveAll() { var request = TimeseriesSaveRequest.builder().build(); - assertThat(request.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL); + assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL); } @Test - void testSaveActionsSaveAll() { - assertThat(TimeseriesSaveRequest.SaveActions.SAVE_ALL).isEqualTo(new TimeseriesSaveRequest.SaveActions(true, true, true)); + void testSaveAllStrategy() { + assertThat(TimeseriesSaveRequest.Strategy.SAVE_ALL).isEqualTo(new TimeseriesSaveRequest.Strategy(true, true, true)); } @Test - void testSaveActionsWsOnly() { - assertThat(TimeseriesSaveRequest.SaveActions.WS_ONLY).isEqualTo(new TimeseriesSaveRequest.SaveActions(false, false, true)); + void testWsOnlyStrategy() { + assertThat(TimeseriesSaveRequest.Strategy.WS_ONLY).isEqualTo(new TimeseriesSaveRequest.Strategy(false, false, true)); } @Test - void testSaveActionsLatestAndWs() { - assertThat(TimeseriesSaveRequest.SaveActions.LATEST_AND_WS).isEqualTo(new TimeseriesSaveRequest.SaveActions(false, true, true)); + void testLatestAndWsStrategy() { + assertThat(TimeseriesSaveRequest.Strategy.LATEST_AND_WS).isEqualTo(new TimeseriesSaveRequest.Strategy(false, true, true)); } @Test - void testSaveActionsSkipAll() { - assertThat(TimeseriesSaveRequest.SaveActions.SKIP_ALL).isEqualTo(new TimeseriesSaveRequest.SaveActions(false, false, false)); + void testSkipAllStrategy() { + assertThat(TimeseriesSaveRequest.Strategy.SKIP_ALL).isEqualTo(new TimeseriesSaveRequest.Strategy(false, false, false)); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index eb2862a0da..53261bf7dd 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -104,10 +104,10 @@ public class TbMsgTimeseriesNode implements TbNode { } long ts = computeTs(msg, config.isUseServerTs()); - TimeseriesSaveRequest.SaveActions saveActions = determineSaveActions(ts, msg.getOriginator().getId()); + TimeseriesSaveRequest.Strategy strategy = determineSaveActions(ts, msg.getOriginator().getId()); // short-circuit - if (!saveActions.saveTimeseries() && !saveActions.saveLatest() && !saveActions.sendWsUpdate()) { + if (!strategy.saveTimeseries() && !strategy.saveLatest() && !strategy.sendWsUpdate()) { ctx.tellSuccess(msg); return; } @@ -135,7 +135,7 @@ public class TbMsgTimeseriesNode implements TbNode { .entityId(msg.getOriginator()) .entries(tsKvEntryList) .ttl(ttl) - .saveActions(saveActions) + .strategy(strategy) .callback(new TelemetryNodeCallback(ctx, msg)) .build()); } @@ -144,19 +144,19 @@ public class TbMsgTimeseriesNode implements TbNode { return ignoreMetadataTs ? System.currentTimeMillis() : msg.getMetaDataTs(); } - private TimeseriesSaveRequest.SaveActions determineSaveActions(long ts, UUID originatorUuid) { + private TimeseriesSaveRequest.Strategy determineSaveActions(long ts, UUID originatorUuid) { if (persistenceSettings instanceof OnEveryMessage) { - return TimeseriesSaveRequest.SaveActions.SAVE_ALL; + return TimeseriesSaveRequest.Strategy.SAVE_ALL; } if (persistenceSettings instanceof WebSocketsOnly) { - return TimeseriesSaveRequest.SaveActions.WS_ONLY; + return TimeseriesSaveRequest.Strategy.WS_ONLY; } if (persistenceSettings instanceof Deduplicate deduplicate) { boolean isFirstMsgInInterval = deduplicate.getDeduplicateStrategy().shouldPersist(ts, originatorUuid); - return isFirstMsgInInterval ? TimeseriesSaveRequest.SaveActions.SAVE_ALL : TimeseriesSaveRequest.SaveActions.SKIP_ALL; + return isFirstMsgInInterval ? TimeseriesSaveRequest.Strategy.SAVE_ALL : TimeseriesSaveRequest.Strategy.SKIP_ALL; } if (persistenceSettings instanceof Advanced advanced) { - return new TimeseriesSaveRequest.SaveActions( + return new TimeseriesSaveRequest.Strategy( advanced.timeseries().shouldPersist(ts, originatorUuid), advanced.latest().shouldPersist(ts, originatorUuid), advanced.webSockets().shouldPersist(ts, originatorUuid) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java index 1270c8cf6c..63432afc37 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java @@ -533,7 +533,7 @@ public class TbMathNodeTest { verify(ctx, timeout(TIMEOUT)).tellSuccess(msgCaptor.capture()); verify(telemetryService, times(1)).saveTimeseries(assertArg(request -> { assertThat(request.getEntries()).size().isOne(); - assertThat(request.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL); + assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL); })); TbMsg resultMsg = msgCaptor.getValue(); @@ -569,7 +569,7 @@ public class TbMathNodeTest { verify(ctx, timeout(TIMEOUT)).tellSuccess(msgCaptor.capture()); verify(telemetryService, times(1)).saveTimeseries(assertArg(request -> { assertThat(request.getEntries()).size().isOne(); - assertThat(request.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL); + assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL); })); TbMsg resultMsg = msgCaptor.getValue(); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java index 71236c97be..02c19ed5fc 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeTest.java @@ -208,7 +208,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { assertThat(request.getEntityId()).isEqualTo(DEVICE_ID); assertThat(request.getEntries()).usingRecursiveFieldByFieldElementComparatorIgnoringFields("ts").containsExactlyElementsOf(expectedList); assertThat(request.getTtl()).isEqualTo(extractTtlAsSeconds(tenantProfile)); - assertThat(request.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL); + assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL); assertThat(request.getCallback()).isInstanceOf(TelemetryNodeCallback.class); })); verify(ctxMock).tellSuccess(msg); @@ -265,7 +265,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { assertThat(request.getEntityId()).isEqualTo(DEVICE_ID); assertThat(request.getEntries()).containsExactlyElementsOf(expectedList); assertThat(request.getTtl()).isEqualTo(config.getDefaultTTL()); - assertThat(request.getSaveActions()).isEqualTo(new TimeseriesSaveRequest.SaveActions(true, false, true)); + assertThat(request.getStrategy()).isEqualTo(new TimeseriesSaveRequest.Strategy(true, false, true)); assertThat(request.getCallback()).isInstanceOf(TelemetryNodeCallback.class); })); verify(ctxMock).tellSuccess(msg); @@ -304,7 +304,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { assertThat(request.getCustomerId()).isNull(); assertThat(request.getEntityId()).isEqualTo(DEVICE_ID); assertThat(request.getTtl()).isEqualTo(expectedTtl); - assertThat(request.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL); + assertThat(request.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL); assertThat(request.getCallback()).isInstanceOf(TelemetryNodeCallback.class); })); } @@ -353,7 +353,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entityId(msg.getOriginator()) .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) - .saveActions(TimeseriesSaveRequest.SaveActions.SAVE_ALL) + .strategy(TimeseriesSaveRequest.Strategy.SAVE_ALL) .build(); node.onMsg(ctxMock, msg); @@ -388,7 +388,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entityId(msg.getOriginator()) .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) - .saveActions(TimeseriesSaveRequest.SaveActions.SAVE_ALL) + .strategy(TimeseriesSaveRequest.Strategy.SAVE_ALL) .build(); node.onMsg(ctxMock, msg); @@ -423,7 +423,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entityId(msg.getOriginator()) .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) - .saveActions(TimeseriesSaveRequest.SaveActions.WS_ONLY) + .strategy(TimeseriesSaveRequest.Strategy.WS_ONLY) .build(); node.onMsg(ctxMock, msg); @@ -462,7 +462,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .entityId(msg.getOriginator()) .entry(new BasicTsKvEntry(123L, new DoubleDataEntry("temperature", 22.3))) .ttl(extractTtlAsSeconds(tenantProfile)) - .saveActions(TimeseriesSaveRequest.SaveActions.SAVE_ALL) + .strategy(TimeseriesSaveRequest.Strategy.SAVE_ALL) .build(); node.onMsg(ctxMock, msg); @@ -499,7 +499,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts1)))) .build()); then(telemetryServiceMock).should().saveTimeseries(assertArg( - actualSaveRequest -> assertThat(actualSaveRequest.getSaveActions()).isEqualTo(TimeseriesSaveRequest.SaveActions.SAVE_ALL) + actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo(TimeseriesSaveRequest.Strategy.SAVE_ALL) )); clearInvocations(telemetryServiceMock); @@ -511,7 +511,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts2)))) .build()); then(telemetryServiceMock).should().saveTimeseries(assertArg( - actualSaveRequest -> assertThat(actualSaveRequest.getSaveActions()).isEqualTo(new TimeseriesSaveRequest.SaveActions(true, false, false)) + actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo(new TimeseriesSaveRequest.Strategy(true, false, false)) )); clearInvocations(telemetryServiceMock); @@ -523,7 +523,7 @@ public class TbMsgTimeseriesNodeTest extends AbstractRuleNodeUpgradeTest { .metaData(new TbMsgMetaData(Map.of("ts", Long.toString(ts3)))) .build()); then(telemetryServiceMock).should().saveTimeseries(assertArg( - actualSaveRequest -> assertThat(actualSaveRequest.getSaveActions()).isEqualTo(new TimeseriesSaveRequest.SaveActions(true, true, false)) + actualSaveRequest -> assertThat(actualSaveRequest.getStrategy()).isEqualTo(new TimeseriesSaveRequest.Strategy(true, true, false)) )); } From a2095636a07af68c083e4fe3bb06e92089082589 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Wed, 22 Jan 2025 10:56:36 +0200 Subject: [PATCH 17/30] Save time series strategies: add max deduplication interval validation --- .../strategy/DeduplicatePersistenceStrategy.java | 6 ++++-- .../strategy/DeduplicatePersistenceStrategyTest.java | 9 ++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java index e4eb982861..513523f8f3 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java @@ -28,14 +28,16 @@ import java.util.UUID; final class DeduplicatePersistenceStrategy implements PersistenceStrategy { private static final int MIN_DEDUPLICATION_INTERVAL_SECS = 1; + private static final int MAX_DEDUPLICATION_INTERVAL_SECS = (int) Duration.ofDays(1L).toSeconds(); private final long deduplicationIntervalMillis; private final LoadingCache> deduplicationCache; @JsonCreator public DeduplicatePersistenceStrategy(@JsonProperty("deduplicationIntervalSecs") int deduplicationIntervalSecs) { - if (deduplicationIntervalSecs < MIN_DEDUPLICATION_INTERVAL_SECS) { - throw new IllegalArgumentException("Deduplication interval must be at least " + MIN_DEDUPLICATION_INTERVAL_SECS + " second(s), was " + deduplicationIntervalSecs + " second(s)"); + if (deduplicationIntervalSecs < MIN_DEDUPLICATION_INTERVAL_SECS || deduplicationIntervalSecs > MAX_DEDUPLICATION_INTERVAL_SECS) { + throw new IllegalArgumentException("Deduplication interval must be at least " + MIN_DEDUPLICATION_INTERVAL_SECS + " second(s) " + + "and at most " + MAX_DEDUPLICATION_INTERVAL_SECS + " second(s), was " + deduplicationIntervalSecs + " second(s)"); } deduplicationIntervalMillis = Duration.ofSeconds(deduplicationIntervalSecs).toMillis(); deduplicationCache = Caffeine.newBuilder() diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java index 3f57a7f3df..8aeda400b4 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java @@ -39,7 +39,14 @@ class DeduplicatePersistenceStrategyTest { void shouldThrowWhenDeduplicationIntervalIsLessThanOneSecond() { assertThatThrownBy(() -> new DeduplicatePersistenceStrategy(0)) .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Deduplication interval must be at least 1 second(s), was 0 second(s)"); + .hasMessageContaining("Deduplication interval must be at least 1 second(s) and at most 86400 second(s), was 0 second(s)"); + } + + @Test + void shouldThrowWhenDeduplicationIntervalIsMoreThan24Hours() { + assertThatThrownBy(() -> new DeduplicatePersistenceStrategy(86401)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Deduplication interval must be at least 1 second(s) and at most 86400 second(s), was 86401 second(s)"); } @Test From 349554f93870ac6b4e34bfa1a0657f8d56759d92 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Wed, 22 Jan 2025 12:17:26 +0200 Subject: [PATCH 18/30] Save time series strategies: dynamically calculate max number of deduplication intervals --- .../engine/telemetry/TbMsgTimeseriesNode.java | 4 +- .../DeduplicatePersistenceStrategy.java | 14 ++++- .../DeduplicatePersistenceStrategyTest.java | 55 +++++++++++++++++++ 3 files changed, 70 insertions(+), 3 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 53261bf7dd..232b8a3d36 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -104,7 +104,7 @@ public class TbMsgTimeseriesNode implements TbNode { } long ts = computeTs(msg, config.isUseServerTs()); - TimeseriesSaveRequest.Strategy strategy = determineSaveActions(ts, msg.getOriginator().getId()); + TimeseriesSaveRequest.Strategy strategy = determineSaveStrategy(ts, msg.getOriginator().getId()); // short-circuit if (!strategy.saveTimeseries() && !strategy.saveLatest() && !strategy.sendWsUpdate()) { @@ -144,7 +144,7 @@ public class TbMsgTimeseriesNode implements TbNode { return ignoreMetadataTs ? System.currentTimeMillis() : msg.getMetaDataTs(); } - private TimeseriesSaveRequest.Strategy determineSaveActions(long ts, UUID originatorUuid) { + private TimeseriesSaveRequest.Strategy determineSaveStrategy(long ts, UUID originatorUuid) { if (persistenceSettings instanceof OnEveryMessage) { return TimeseriesSaveRequest.Strategy.SAVE_ALL; } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java index 513523f8f3..601328c304 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java @@ -30,6 +30,9 @@ final class DeduplicatePersistenceStrategy implements PersistenceStrategy { private static final int MIN_DEDUPLICATION_INTERVAL_SECS = 1; private static final int MAX_DEDUPLICATION_INTERVAL_SECS = (int) Duration.ofDays(1L).toSeconds(); + private static final int MAX_TOTAL_INTERVALS_DURATION_SECS = (int) Duration.ofDays(2L).toSeconds(); + private static final int MAX_NUMBER_OF_INTERVALS = 100; + private final long deduplicationIntervalMillis; private final LoadingCache> deduplicationCache; @@ -43,10 +46,19 @@ final class DeduplicatePersistenceStrategy implements PersistenceStrategy { deduplicationCache = Caffeine.newBuilder() .softValues() .expireAfterAccess(Duration.ofSeconds(deduplicationIntervalSecs * 10L)) - .maximumSize(20L) + .maximumSize(calculateMaxNumberOfDeduplicationIntervals(deduplicationIntervalSecs)) .build(__ -> Sets.newConcurrentHashSet()); } + /** + * Calculates the maximum number of deduplication intervals we will store in the cache. + * We limit retention to two days to avoid stale data and cap it at 100 intervals to manage memory usage. + */ + private static long calculateMaxNumberOfDeduplicationIntervals(int deduplicationIntervalSecs) { + int numberOfDeduplicationIntervals = MAX_TOTAL_INTERVALS_DURATION_SECS / deduplicationIntervalSecs; + return Math.min(numberOfDeduplicationIntervals, MAX_NUMBER_OF_INTERVALS); + } + @JsonProperty("deduplicationIntervalSecs") public long getDeduplicationIntervalSecs() { return Duration.ofMillis(deduplicationIntervalMillis).toSeconds(); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java index 8aeda400b4..1a6bdc831d 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java @@ -15,10 +15,14 @@ */ package org.thingsboard.rule.engine.telemetry.strategy; +import com.github.benmanes.caffeine.cache.LoadingCache; +import com.github.benmanes.caffeine.cache.Policy; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.springframework.test.util.ReflectionTestUtils; import java.time.Duration; +import java.util.Set; import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; @@ -49,6 +53,57 @@ class DeduplicatePersistenceStrategyTest { .hasMessageContaining("Deduplication interval must be at least 1 second(s) and at most 86400 second(s), was 86401 second(s)"); } + @Test + void shouldNotAllowMoreThan100DeduplicationIntervals() { + // GIVEN + int deduplicationIntervalSecs = 1; // min deduplication interval duration + + // WHEN + strategy = new DeduplicatePersistenceStrategy(deduplicationIntervalSecs); + + // THEN + var deduplicationCache = (LoadingCache>) ReflectionTestUtils.getField(strategy, "deduplicationCache"); + + assertThat(deduplicationCache.policy().eviction()) + .isPresent() + .map(Policy.Eviction::getMaximum) + .hasValue(100L); + } + + @Test + void shouldCalculateMaxIntervalsAsTwoDaysDividedByIntervalDuration() { + // GIVEN + int deduplicationIntervalSecs = (int) Duration.ofHours(1L).toSeconds(); + + // WHEN + strategy = new DeduplicatePersistenceStrategy(deduplicationIntervalSecs); + + // THEN + var deduplicationCache = (LoadingCache>) ReflectionTestUtils.getField(strategy, "deduplicationCache"); + + assertThat(deduplicationCache.policy().eviction()) + .isPresent() + .map(Policy.Eviction::getMaximum) + .hasValue(48L); + } + + @Test + void shouldKeepAtLeastTwoDeduplicationIntervals() { + // GIVEN + int deduplicationIntervalSecs = (int) Duration.ofDays(1L).toSeconds(); // max deduplication interval duration + + // WHEN + strategy = new DeduplicatePersistenceStrategy(deduplicationIntervalSecs); + + // THEN + var deduplicationCache = (LoadingCache>) ReflectionTestUtils.getField(strategy, "deduplicationCache"); + + assertThat(deduplicationCache.policy().eviction()) + .isPresent() + .map(Policy.Eviction::getMaximum) + .hasValue(2L); + } + @Test void shouldReturnTrueForFirstCallInInterval() { long ts = 1_000_000L; From 0131358d3ba3b5e37c203c9500f7fc7ef477335e Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Wed, 22 Jan 2025 15:10:56 +0200 Subject: [PATCH 19/30] Save time series strategies: dynamically calculate interval expire after access --- .../DeduplicatePersistenceStrategy.java | 18 ++++++- .../DeduplicatePersistenceStrategyTest.java | 51 +++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java index 601328c304..868e0d8ca4 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategy.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.LoadingCache; import com.google.common.collect.Sets; +import com.google.common.primitives.Longs; import java.time.Duration; import java.util.Set; @@ -30,6 +31,10 @@ final class DeduplicatePersistenceStrategy implements PersistenceStrategy { private static final int MIN_DEDUPLICATION_INTERVAL_SECS = 1; private static final int MAX_DEDUPLICATION_INTERVAL_SECS = (int) Duration.ofDays(1L).toSeconds(); + private static final long MIN_INTERVAL_EXPIRY_MILLIS = Duration.ofMinutes(10L).toMillis(); + private static final int INTERVAL_EXPIRY_FACTOR = 10; + private static final long MAX_INTERVAL_EXPIRY_MILLIS = Duration.ofDays(2L).toMillis(); + private static final int MAX_TOTAL_INTERVALS_DURATION_SECS = (int) Duration.ofDays(2L).toSeconds(); private static final int MAX_NUMBER_OF_INTERVALS = 100; @@ -45,11 +50,22 @@ final class DeduplicatePersistenceStrategy implements PersistenceStrategy { deduplicationIntervalMillis = Duration.ofSeconds(deduplicationIntervalSecs).toMillis(); deduplicationCache = Caffeine.newBuilder() .softValues() - .expireAfterAccess(Duration.ofSeconds(deduplicationIntervalSecs * 10L)) + .expireAfterAccess(calculateExpireAfterAccess(deduplicationIntervalSecs)) .maximumSize(calculateMaxNumberOfDeduplicationIntervals(deduplicationIntervalSecs)) .build(__ -> Sets.newConcurrentHashSet()); } + /** + * Calculates the expire-after-access duration. By default, we keep each deduplication interval + * alive for 10 “iterations” (interval duration × 10). However, we never let this drop below + * 10 minutes to ensure adequate retention for small intervals, nor exceed 48 hours to prevent + * storing stale data in memory. + */ + private static Duration calculateExpireAfterAccess(int deduplicationIntervalSecs) { + long desiredExpiryMillis = Duration.ofSeconds(deduplicationIntervalSecs).toMillis() * INTERVAL_EXPIRY_FACTOR; + return Duration.ofMillis(Longs.constrainToRange(desiredExpiryMillis, MIN_INTERVAL_EXPIRY_MILLIS, MAX_INTERVAL_EXPIRY_MILLIS)); + } + /** * Calculates the maximum number of deduplication intervals we will store in the cache. * We limit retention to two days to avoid stale data and cap it at 100 intervals to manage memory usage. diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java index 1a6bdc831d..1ae2b367a5 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/telemetry/strategy/DeduplicatePersistenceStrategyTest.java @@ -53,6 +53,57 @@ class DeduplicatePersistenceStrategyTest { .hasMessageContaining("Deduplication interval must be at least 1 second(s) and at most 86400 second(s), was 86401 second(s)"); } + @Test + void shouldUseAtLeastTenMinutesForExpireAfterAccess() { + // GIVEN + int deduplicationIntervalSecs = 1; // min deduplication interval duration + + // WHEN + strategy = new DeduplicatePersistenceStrategy(deduplicationIntervalSecs); + + // THEN + var deduplicationCache = (LoadingCache>) ReflectionTestUtils.getField(strategy, "deduplicationCache"); + + assertThat(deduplicationCache.policy().expireAfterAccess()) + .isPresent() + .map(Policy.FixedExpiration::getExpiresAfter) + .hasValue(Duration.ofMinutes(10L)); + } + + @Test + void shouldCalculateExpireAfterAccessAsIntervalDurationMultipliedByTen() { + // GIVEN + int deduplicationIntervalSecs = (int) Duration.ofHours(1L).toSeconds(); // max deduplication interval duration + + // WHEN + strategy = new DeduplicatePersistenceStrategy(deduplicationIntervalSecs); + + // THEN + var deduplicationCache = (LoadingCache>) ReflectionTestUtils.getField(strategy, "deduplicationCache"); + + assertThat(deduplicationCache.policy().expireAfterAccess()) + .isPresent() + .map(Policy.FixedExpiration::getExpiresAfter) + .hasValue(Duration.ofHours(10L)); + } + + @Test + void shouldUseAtMostTwoDaysForExpireAfterAccess() { + // GIVEN + int deduplicationIntervalSecs = (int) Duration.ofDays(1L).toSeconds(); // max deduplication interval duration + + // WHEN + strategy = new DeduplicatePersistenceStrategy(deduplicationIntervalSecs); + + // THEN + var deduplicationCache = (LoadingCache>) ReflectionTestUtils.getField(strategy, "deduplicationCache"); + + assertThat(deduplicationCache.policy().expireAfterAccess()) + .isPresent() + .map(Policy.FixedExpiration::getExpiresAfter) + .hasValue(Duration.ofDays(2L)); + } + @Test void shouldNotAllowMoreThan100DeduplicationIntervals() { // GIVEN From 2d1ead5f544520c4a35c7149e19399f181dd4d65 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 22 Jan 2025 15:43:59 +0200 Subject: [PATCH 20/30] UI: Add persistence settings in save ts rule node --- .../action/action-rule-node-config.module.ts | 10 +- ...ced-persistence-setting-row.component.html | 39 ++++++ ...anced-persistence-setting-row.component.ts | 114 ++++++++++++++++++ ...dvanced-persistence-setting.component.html | 31 +++++ .../advanced-persistence-setting.component.ts | 83 +++++++++++++ .../action/timeseries-config.component.html | 40 ++++++ .../action/timeseries-config.component.ts | 104 ++++++++++++++-- .../action/timeseries-config.models.ts | 78 ++++++++++++ .../common/time-unit-input.component.html | 3 +- .../assets/locale/locale.constant-en_US.json | 19 +++ 10 files changed, 511 insertions(+), 10 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.html create mode 100644 ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.html create mode 100644 ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/action-rule-node-config.module.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/action-rule-node-config.module.ts index 17307cb341..caec009629 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/action-rule-node-config.module.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/action-rule-node-config.module.ts @@ -42,6 +42,12 @@ import { DeleteAttributesConfigComponent } from './delete-attributes-config.comp import { MathFunctionConfigComponent } from './math-function-config.component'; import { DeviceStateConfigComponent } from './device-state-config.component'; import { SendRestApiCallReplyConfigComponent } from './send-rest-api-call-reply-config.component'; +import { + AdvancedPersistenceSettingComponent +} from '@home/components/rule-node/action/advanced-persistence-setting.component'; +import { + AdvancedPersistenceSettingRowComponent +} from '@home/components/rule-node/action/advanced-persistence-setting-row.component'; @NgModule({ declarations: [ @@ -67,7 +73,9 @@ import { SendRestApiCallReplyConfigComponent } from './send-rest-api-call-reply- PushToEdgeConfigComponent, PushToCloudConfigComponent, MathFunctionConfigComponent, - DeviceStateConfigComponent + DeviceStateConfigComponent, + AdvancedPersistenceSettingComponent, + AdvancedPersistenceSettingRowComponent, ], imports: [ CommonModule, diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.html new file mode 100644 index 0000000000..038fe41bb2 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.html @@ -0,0 +1,39 @@ + +
+
{{ title }}
+ + rule-node-config.save-time-series.strategy + + @for (strategy of persistenceStrategies; track strategy) { + {{ PersistenceTypeTranslationMap.get(strategy) | translate }} + } + + + @if(persistenceSettingRowForm.get('type').value === PersistenceType.DEDUPLICATE) { + + + } +
diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.ts new file mode 100644 index 0000000000..64ea045f8f --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.ts @@ -0,0 +1,114 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input } from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator +} from '@angular/forms'; +import { + AdvancedPersistenceConfig, + defaultAdvancedPersistenceConfig, + maxDeduplicateTimeSecs, + PersistenceType, + PersistenceTypeTranslationMap +} from '@home/components/rule-node/action/timeseries-config.models'; +import { isDefinedAndNotNull } from '@core/utils'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; + +@Component({ + selector: 'tb-advanced-persistence-setting-row', + templateUrl: './advanced-persistence-setting-row.component.html', + providers: [{ + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => AdvancedPersistenceSettingRowComponent), + multi: true + },{ + provide: NG_VALIDATORS, + useExisting: forwardRef(() => AdvancedPersistenceSettingRowComponent), + multi: true + }] +}) +export class AdvancedPersistenceSettingRowComponent implements ControlValueAccessor, Validator { + + @Input() + title: string; + + persistenceSettingRowForm = this.fb.group({ + type: [defaultAdvancedPersistenceConfig.type], + deduplicationIntervalSecs: [{value: 60, disabled: true}] + }); + + PersistenceType = PersistenceType; + persistenceStrategies = [PersistenceType.ON_EVERY_MESSAGE, PersistenceType.DEDUPLICATE, PersistenceType.SKIP]; + PersistenceTypeTranslationMap = PersistenceTypeTranslationMap; + + maxDeduplicateTime = maxDeduplicateTimeSecs; + + private propagateChange: (value: any) => void = () => {}; + + constructor(private fb: FormBuilder) { + this.persistenceSettingRowForm.get('type').valueChanges.pipe( + takeUntilDestroyed() + ).subscribe(() => this.updatedValidation()); + + this.persistenceSettingRowForm.valueChanges.pipe( + takeUntilDestroyed() + ).subscribe((value) => this.propagateChange(value)); + } + + registerOnChange(fn: any) { + this.propagateChange = fn; + } + + registerOnTouched(_fn: any) { + } + + setDisabledState(isDisabled: boolean) { + if (isDisabled) { + this.persistenceSettingRowForm.disable({emitEvent: false}); + } else { + this.persistenceSettingRowForm.enable({emitEvent: false}); + this.updatedValidation(); + } + } + + validate(): ValidationErrors | null { + return this.persistenceSettingRowForm.valid ? null : { + persistenceSettingRow: false + }; + } + + writeValue(value: AdvancedPersistenceConfig) { + if (isDefinedAndNotNull(value)) { + this.persistenceSettingRowForm.patchValue(value, {emitEvent: false}); + } else { + this.persistenceSettingRowForm.patchValue(defaultAdvancedPersistenceConfig); + } + } + + private updatedValidation() { + if (this.persistenceSettingRowForm.get('type').value === PersistenceType.DEDUPLICATE) { + this.persistenceSettingRowForm.get('deduplicationIntervalSecs').enable({emitEvent: false}); + } else { + this.persistenceSettingRowForm.get('deduplicationIntervalSecs').disable({emitEvent: false}) + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.html new file mode 100644 index 0000000000..eb0f05bec2 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.html @@ -0,0 +1,31 @@ + +
+ + + +
diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.ts new file mode 100644 index 0000000000..5c9930fbb7 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting.component.ts @@ -0,0 +1,83 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator +} from '@angular/forms'; +import { Component, forwardRef } from '@angular/core'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { AdvancedPersistenceStrategy } from '@home/components/rule-node/action/timeseries-config.models'; + +@Component({ + selector: 'tb-advanced-persistence-settings', + templateUrl: './advanced-persistence-setting.component.html', + providers: [{ + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => AdvancedPersistenceSettingComponent), + multi: true + },{ + provide: NG_VALIDATORS, + useExisting: forwardRef(() => AdvancedPersistenceSettingComponent), + multi: true + }] +}) +export class AdvancedPersistenceSettingComponent implements ControlValueAccessor, Validator { + + persistenceForm = this.fb.group({ + timeseries: [null], + latest: [null], + webSockets: [null] + }); + + private propagateChange: (value: any) => void = () => {}; + + constructor(private fb: FormBuilder) { + this.persistenceForm.valueChanges.pipe( + takeUntilDestroyed() + ).subscribe(value => this.propagateChange(value)); + } + + registerOnChange(fn: any) { + this.propagateChange = fn; + } + + registerOnTouched(_fn: any) { + } + + setDisabledState(isDisabled: boolean) { + if (isDisabled) { + this.persistenceForm.disable({emitEvent: false}); + } else { + this.persistenceForm.enable({emitEvent: false}); + } + } + + validate(): ValidationErrors | null { + return this.persistenceForm.valid ? null : { + persistenceForm: false + }; + } + + writeValue(value: AdvancedPersistenceStrategy) { + this.persistenceForm.patchValue(value, {emitEvent: false}); + } + +} diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html index aa70914fa9..47655c1285 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html @@ -16,6 +16,46 @@ -->
+
+
+
+ rule-node-config.save-time-series.persistence-settings +
+ + {{ 'rule-node-config.basic-mode' | translate}} + {{ 'rule-node-config.advanced-mode' | translate }} + +
+ @if(!timeseriesConfigForm.get('persistenceSettings.isAdvanced').value) { + + rule-node-config.save-time-series.strategy + + @for (strategy of persistenceStrategies; track strategy) { + {{ PersistenceTypeTranslationMap.get(strategy) | translate }} + } + + + + @if(timeseriesConfigForm.get('persistenceSettings.type').value === PersistenceType.DEDUPLICATE) { + + + } + } + @else { + + } +
diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts index 0c5283f54a..a834d0d43f 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts @@ -15,8 +15,18 @@ /// import { Component } from '@angular/core'; -import { UntypedFormBuilder, UntypedFormGroup, Validators } from '@angular/forms'; -import { RuleNodeConfiguration, RuleNodeConfigurationComponent } from '@shared/models/rule-node.models'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { RuleNodeConfigurationComponent } from '@shared/models/rule-node.models'; +import { + defaultAdvancedPersistenceStrategy, + maxDeduplicateTimeSecs, + PersistenceSettings, + PersistenceSettingsForm, + PersistenceType, + PersistenceTypeTranslationMap, + TimeseriesNodeConfiguration, + TimeseriesNodeConfigurationForm +} from '@home/components/rule-node/action/timeseries-config.models'; @Component({ selector: 'tb-action-node-timeseries-config', @@ -25,20 +35,98 @@ import { RuleNodeConfiguration, RuleNodeConfigurationComponent } from '@shared/m }) export class TimeseriesConfigComponent extends RuleNodeConfigurationComponent { - timeseriesConfigForm: UntypedFormGroup; + timeseriesConfigForm: FormGroup; - constructor(private fb: UntypedFormBuilder) { + PersistenceType = PersistenceType; + persistenceStrategies = [PersistenceType.ON_EVERY_MESSAGE, PersistenceType.DEDUPLICATE, PersistenceType.WEBSOCKETS_ONLY]; + PersistenceTypeTranslationMap = PersistenceTypeTranslationMap; + + maxDeduplicateTime = maxDeduplicateTimeSecs + + constructor(private fb: FormBuilder) { super(); } - protected configForm(): UntypedFormGroup { + protected configForm(): FormGroup { return this.timeseriesConfigForm; } - protected onConfigurationSet(configuration: RuleNodeConfiguration) { + protected validatorTriggers(): string[] { + return ['persistenceSettings.isAdvanced', 'persistenceSettings.type']; + } + + protected prepareInputConfig(config: TimeseriesNodeConfiguration): TimeseriesNodeConfigurationForm { + let persistenceSettings: PersistenceSettingsForm; + if (config?.persistenceSettings) { + const isAdvanced = config?.persistenceSettings?.type === PersistenceType.ADVANCED; + persistenceSettings = { + ...config.persistenceSettings, + isAdvanced: isAdvanced, + type: isAdvanced ? PersistenceType.ON_EVERY_MESSAGE : config.persistenceSettings.type, + advanced: isAdvanced ? config.persistenceSettings : defaultAdvancedPersistenceStrategy + } + } else { + persistenceSettings = { + type: PersistenceType.ON_EVERY_MESSAGE, + isAdvanced: false, + deduplicationIntervalSecs: 10, + advanced: defaultAdvancedPersistenceStrategy + }; + } + return { + ...config, + persistenceSettings: persistenceSettings + } + } + + protected prepareOutputConfig(config: TimeseriesNodeConfigurationForm): TimeseriesNodeConfiguration { + let persistenceSettings: PersistenceSettings; + if (config.persistenceSettings.isAdvanced) { + persistenceSettings = { + ...config.persistenceSettings.advanced, + type: PersistenceType.ADVANCED + }; + } else { + persistenceSettings = { + type: config.persistenceSettings.type, + deduplicationIntervalSecs: config.persistenceSettings?.deduplicationIntervalSecs + }; + } + return { + ...config, + persistenceSettings + }; + } + + protected onConfigurationSet(config: TimeseriesNodeConfigurationForm) { this.timeseriesConfigForm = this.fb.group({ - defaultTTL: [configuration ? configuration.defaultTTL : null, [Validators.required, Validators.min(0)]], - useServerTs: [configuration ? configuration.useServerTs : false, []] + persistenceSettings: this.fb.group({ + isAdvanced: [config?.persistenceSettings?.isAdvanced ?? false], + type: [config?.persistenceSettings?.type ?? PersistenceType.ON_EVERY_MESSAGE], + deduplicationIntervalSecs: [ + {value: config?.persistenceSettings?.deduplicationIntervalSecs ?? 10, disabled: true}, + [Validators.required, Validators.max(maxDeduplicateTimeSecs)] + ], + advanced: [{value: null, disabled: true}] + }), + defaultTTL: [config?.defaultTTL ?? null, [Validators.required, Validators.min(0)]], + useServerTs: [config?.useServerTs ?? false] }); } + + protected updateValidators(emitEvent: boolean, _trigger?: string) { + const persistenceForm = this.timeseriesConfigForm.get('persistenceSettings') as FormGroup; + const isAdvanced: boolean = persistenceForm.get('isAdvanced').value; + const type: PersistenceType = persistenceForm.get('type').value; + if (!isAdvanced && type === PersistenceType.DEDUPLICATE) { + persistenceForm.get('deduplicationIntervalSecs').enable({emitEvent}); + } else { + persistenceForm.get('deduplicationIntervalSecs').disable({emitEvent}); + } + if (isAdvanced) { + persistenceForm.get('advanced').enable({emitEvent}); + } else { + persistenceForm.get('advanced').disable({emitEvent}); + } + } } diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts new file mode 100644 index 0000000000..241787d511 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts @@ -0,0 +1,78 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { DAY, SECOND } from '@shared/models/time/time.models'; + +export const maxDeduplicateTimeSecs = DAY / SECOND + 1; + +export interface TimeseriesNodeConfiguration { + persistenceSettings: PersistenceSettings; + defaultTTL: number; + useServerTs: boolean; +} + +export interface TimeseriesNodeConfigurationForm extends Omit { + persistenceSettings: PersistenceSettingsForm +} + +export type PersistenceSettings = BasicPersistenceSettings & Partial & Partial; + +export type PersistenceSettingsForm = Omit & { + isAdvanced: boolean; + advanced?: Partial; + type: PersistenceType; +}; + +export enum PersistenceType { + ON_EVERY_MESSAGE = 'ON_EVERY_MESSAGE', + DEDUPLICATE = 'DEDUPLICATE', + WEBSOCKETS_ONLY = 'WEBSOCKETS_ONLY', + ADVANCED = 'ADVANCED', + SKIP = 'SKIP' +} + +export const PersistenceTypeTranslationMap = new Map([ + [PersistenceType.ON_EVERY_MESSAGE, 'rule-node-config.save-time-series.strategy-type.every-message'], + [PersistenceType.DEDUPLICATE, 'rule-node-config.save-time-series.strategy-type.deduplicate'], + [PersistenceType.WEBSOCKETS_ONLY, 'rule-node-config.save-time-series.strategy-type.web-sockets-only'], + [PersistenceType.SKIP, 'rule-node-config.save-time-series.strategy-type.skip'], +]) + +export interface BasicPersistenceSettings { + type: PersistenceType; +} + +export interface DeduplicatePersistenceStrategy extends BasicPersistenceSettings{ + deduplicationIntervalSecs: number; +} + +export interface AdvancedPersistenceStrategy extends BasicPersistenceSettings{ + timeseries: AdvancedPersistenceConfig; + latest: AdvancedPersistenceConfig; + webSockets: AdvancedPersistenceConfig; +} + +export type AdvancedPersistenceConfig = WithOptional; + +export const defaultAdvancedPersistenceConfig: AdvancedPersistenceConfig = { + type: PersistenceType.ON_EVERY_MESSAGE +} + +export const defaultAdvancedPersistenceStrategy: Omit = { + timeseries: defaultAdvancedPersistenceConfig, + latest: defaultAdvancedPersistenceConfig, + webSockets: defaultAdvancedPersistenceConfig, +} diff --git a/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html index 79caab4cb7..1fd0a60159 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html @@ -16,12 +16,13 @@ -->
- + {{ labelText }}
+ {{ requiredText }} diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index b6aa720a53..f19336d6ad 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -5068,6 +5068,25 @@ "units": "Units", "tell-failure-aws-lambda": "Tell Failure if AWS Lambda function execution raises exception", "tell-failure-aws-lambda-hint": "Rule node forces failure of message processing if AWS Lambda function execution raises exception.", + "basic-mode": "Basic", + "advanced-mode": "Advanced", + "save-time-series": { + "persistence-settings": "Persistence settings", + "persistence-settings-hint": "Persistence settings hint", + "strategy": "Strategy", + "deduplication-interval": "Deduplication interval", + "deduplication-interval-required": "Deduplication interval is required", + "deduplication-interval-min-max-range": "Deduplication interval should be at least 1 second and at most 1 day", + "strategy-type": { + "every-message": "On every message", + "skip": "Skip", + "deduplicate": "Deduplicate", + "web-sockets-only": "WebSockets only" + }, + "time-series": "Time series", + "latest": "Latest", + "web-sockets": "WebSockets" + }, "key-val": { "key": "Key", "value": "Value", From 3dfcc318e070675df6eb95e2470c4b76fe132516 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Wed, 22 Jan 2025 16:21:47 +0200 Subject: [PATCH 21/30] Save time series strategies: add information about strategies in node description; refactor node description --- .../engine/telemetry/TbMsgTimeseriesNode.java | 55 +++++++++++++++---- 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 232b8a3d36..31deddf63b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -57,18 +57,49 @@ import static org.thingsboard.server.common.data.msg.TbMsgType.POST_TELEMETRY_RE type = ComponentType.ACTION, name = "save time series", configClazz = TbMsgTimeseriesNodeConfiguration.class, - nodeDescription = "Saves time series data", - nodeDetails = "Saves time series telemetry data based on configurable TTL parameter. Expects messages with 'POST_TELEMETRY_REQUEST' message type. " + - "Timestamp in milliseconds will be taken from metadata.ts, otherwise 'now' message timestamp will be applied. " + - "Allows stopping updating values for incoming keys in the latest ts_kv table if 'skipLatestPersistence' is set to true.\n " + - "
" + - "Enable 'useServerTs' param to use the timestamp of the message processing instead of the timestamp from the message. " + - "Useful for all sorts of sequential processing if you merge messages from multiple sources (devices, assets, etc).\n" + - "
" + - "In the case of sequential processing, the platform guarantees that the messages are processed in the order of their submission to the queue. " + - "However, the timestamp of the messages originated by multiple devices/servers may be unsynchronized long before they are pushed to the queue. " + - "The DB layer has certain optimizations to ignore the updates of the \"attributes\" and \"latest values\" tables if the new record has a timestamp that is older than the previous record. " + - "So, to make sure that all the messages will be processed correctly, one should enable this parameter for sequential message processing scenarios.", + nodeDescription = """ + Saves time series data with a configurable TTL and according to configured persistence strategies. + """, + nodeDetails = """ + Node performs three actions: +
    +
  • Time series: save time series data to a ts_kv table in a DB.
    • +
  • Latest values: save time series data to a ts_kv_latest table in a DB.
    • +
  • WebSockets: notify WebSockets subscriptions about time series data updates.
    • +
+ + For each action, three persistence strategies are available: +
    +
  • On every message: perform the action for every message.
    • +
  • Deduplicate: perform the action only for the first message from a particular originator within a configurable interval.
    • +
  • Skip: never perform the action.
    • +
+ + Persistence strategies are configured using persistence settings, which support two modes: +
    +
  • Basic +
      +
    • On every message: applies the "On every message" strategy to all actions.
      • +
    • Deduplicate: applies the "Deduplicate" strategy (with a specified interval) to all actions.
      • +
    • WebSockets only: applies the "Skip" strategy to Time series and Latest values, and the "On every message" strategy to WebSockets.
      • +
    +
    • +
  • Advanced: configure each action’s strategy independently.
    • +
+ + By default, the timestamp is taken from metadata.ts. You can enable + Use server timestamp to always use the current server time instead. This is particularly + useful in sequential processing scenarios where messages may arrive with out-of-order timestamps from + multiple sources. Note that the DB layer may ignore older records for attributes and latest values, + so enabling Use server timestamp can ensure correct ordering. +

+ The TTL is taken first from metadata.TTL. If absent, the node configuration’s default + TTL is used. If neither is set, the tenant profile default applies. +

+ This node expects messages of type POST_TELEMETRY_REQUEST. +

+ Output connections: Success, Failure. + """, uiResources = {"static/rulenode/rulenode-core-config.js"}, configDirective = "tbActionNodeTimeseriesConfig", icon = "file_upload", From e57746167dcdd582e4097e11fc8b6a9d06bbb0b0 Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Thu, 23 Jan 2025 10:55:54 +0200 Subject: [PATCH 22/30] Update locale.constant-en_US.json --- ui-ngx/src/assets/locale/locale.constant-en_US.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index f19336d6ad..b5903a8fb6 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -4907,7 +4907,7 @@ "alarm-severity-pattern-hint": "Use ${metadataKey} for value from metadata, $[messageKey] for value from message body. Alarm severity should be system (CRITICAL, MAJOR etc.)", "output-node-name-hint": "The rule node name corresponds to the relation type of the output message, and it is used to forward messages to other rule nodes in the caller rule chain.", "use-server-ts": "Use server timestamp", - "use-server-ts-hint": "Rule node will use the timestamp of message processing instead of the timestamp from the message. Useful for all sorts of sequential processing if you merge messages from multiple sources (devices, assets, etc).", + "use-server-ts-hint": "Use the server’s current timestamp for time series data that lacks an explicit timestamp. This helps maintain proper ordering when processing messages from multiple sources or when messages arrive out of sequence.", "kv-map-pattern-hint": "All input fields support templatization. Use $[messageKey] to extract value from the message and ${metadataKey} to extract value from the metadata.", "kv-map-single-pattern-hint": "Input field support templatization. Use $[messageKey] to extract value from the message and ${metadataKey} to extract value from the metadata.", "shared-scope": "Shared scope", @@ -5072,7 +5072,7 @@ "advanced-mode": "Advanced", "save-time-series": { "persistence-settings": "Persistence settings", - "persistence-settings-hint": "Persistence settings hint", + "persistence-settings-hint": "Define how and when time series data is saved. In Basic mode, apply a single persistence strategy to all actions or enable only WebSockets updates. Advanced mode allows you to configure individual persistence strategies for each action.", "strategy": "Strategy", "deduplication-interval": "Deduplication interval", "deduplication-interval-required": "Deduplication interval is required", From 80593f4b52d09703e397bb20799d9501baed1f60 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 23 Jan 2025 14:05:41 +0200 Subject: [PATCH 23/30] UI: Fixed validation and style in Save ts rule node config --- ...ced-persistence-setting-row.component.html | 1 + .../action/timeseries-config.component.html | 15 ++++++------- .../action/timeseries-config.component.ts | 8 +++---- .../action/timeseries-config.models.ts | 2 +- .../common/time-unit-input.component.html | 2 +- .../common/time-unit-input.component.ts | 21 ++++++++++++++----- 6 files changed, 31 insertions(+), 18 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.html index 038fe41bb2..2bb51ad504 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/advanced-persistence-setting-row.component.html @@ -32,6 +32,7 @@ requiredText="{{ 'rule-node-config.save-time-series.deduplication-interval-required' | translate }}" minErrorText="{{ 'rule-node-config.save-time-series.deduplication-interval-min-max-range' | translate }}" maxErrorText="{{ 'rule-node-config.save-time-series.deduplication-interval-min-max-range' | translate }}" + [minTime]="1" [maxTime]="maxDeduplicateTime" formControlName="deduplicationIntervalSecs"> diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html index 47655c1285..eb2811c8f8 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html @@ -45,6 +45,7 @@ minErrorText="{{ 'rule-node-config.save-time-series.deduplication-interval-min-max-range' | translate }}" maxErrorText="{{ 'rule-node-config.save-time-series.deduplication-interval-min-max-range' | translate }}" [maxTime]="maxDeduplicateTime" + [minTime]="1" formControlName="deduplicationIntervalSecs"> } @@ -56,12 +57,18 @@ > } -
+
rule-node-config.advanced-settings +
+ + {{ 'rule-node-config.use-server-ts' | translate }} + +
-
- - {{ 'rule-node-config.use-server-ts' | translate }} - -
diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts index a834d0d43f..4a061d3193 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.ts @@ -60,16 +60,16 @@ export class TimeseriesConfigComponent extends RuleNodeConfigurationComponent { if (config?.persistenceSettings) { const isAdvanced = config?.persistenceSettings?.type === PersistenceType.ADVANCED; persistenceSettings = { - ...config.persistenceSettings, - isAdvanced: isAdvanced, type: isAdvanced ? PersistenceType.ON_EVERY_MESSAGE : config.persistenceSettings.type, + isAdvanced: isAdvanced, + deduplicationIntervalSecs: config.persistenceSettings?.deduplicationIntervalSecs ?? 60, advanced: isAdvanced ? config.persistenceSettings : defaultAdvancedPersistenceStrategy } } else { persistenceSettings = { type: PersistenceType.ON_EVERY_MESSAGE, isAdvanced: false, - deduplicationIntervalSecs: 10, + deduplicationIntervalSecs: 60, advanced: defaultAdvancedPersistenceStrategy }; } @@ -104,7 +104,7 @@ export class TimeseriesConfigComponent extends RuleNodeConfigurationComponent { isAdvanced: [config?.persistenceSettings?.isAdvanced ?? false], type: [config?.persistenceSettings?.type ?? PersistenceType.ON_EVERY_MESSAGE], deduplicationIntervalSecs: [ - {value: config?.persistenceSettings?.deduplicationIntervalSecs ?? 10, disabled: true}, + {value: config?.persistenceSettings?.deduplicationIntervalSecs ?? 60, disabled: true}, [Validators.required, Validators.max(maxDeduplicateTimeSecs)] ], advanced: [{value: null, disabled: true}] diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts index 241787d511..f70e8548b1 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.models.ts @@ -16,7 +16,7 @@ import { DAY, SECOND } from '@shared/models/time/time.models'; -export const maxDeduplicateTimeSecs = DAY / SECOND + 1; +export const maxDeduplicateTimeSecs = DAY / SECOND; export interface TimeseriesNodeConfiguration { persistenceSettings: PersistenceSettings; diff --git a/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html index 1fd0a60159..9f65a2d9fc 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.html @@ -33,7 +33,7 @@ {{ maxErrorText }} - + rule-node-config.units @for (timeUnit of timeUnits; track timeUnit) { diff --git a/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts index 3dc0c00124..08d501477c 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts @@ -22,7 +22,8 @@ import { NG_VALIDATORS, NG_VALUE_ACCESSOR, ValidationErrors, - Validator, Validators + Validator, + Validators } from '@angular/forms'; import { TimeUnit, timeUnitTranslations } from '../rule-node-config.models'; import { isDefinedAndNotNull, isNumeric } from '@core/utils'; @@ -60,6 +61,10 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, @Input() requiredText: string; + @Input() + @coerceNumber() + minTime = 0; + @Input() minErrorText: string; @@ -75,7 +80,7 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, timeUnitTranslations = timeUnitTranslations; timeInputForm = this.fb.group({ - time: [0, Validators.min(0)], + time: [0], timeUnit: [TimeUnit.SECONDS] }); @@ -97,7 +102,7 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, ngOnInit() { if(this.required || this.maxTime) { const timeControl = this.timeInputForm.get('time'); - const validators = []; + const validators = [Validators.pattern(/^\d*$/)]; if (this.required) { validators.push(Validators.required); } @@ -106,6 +111,9 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, Validators.max(Math.floor(this.maxTime / this.timeIntervalsInSec.get(this.timeInputForm.get('timeUnit').value)))(control) ); } + if (isDefinedAndNotNull(this.minTime)) { + validators.push(Validators.min(this.minTime)); + } timeControl.setValidators(validators); timeControl.updateValueAndValidity({ emitEvent: false }); @@ -137,6 +145,9 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, this.timeInputForm.disable({emitEvent: false}); } else { this.timeInputForm.enable({emitEvent: false}); + if(this.timeInputForm.invalid) { + setTimeout(() => this.updatedModel(this.timeInputForm.value, true)) + } } } @@ -161,9 +172,9 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, }; } - private updatedModel(value: Partial) { + private updatedModel(value: Partial, forceUpdated = false) { const time = value.time * this.timeIntervalsInSec.get(value.timeUnit); - if (this.modelValue !== time) { + if (this.modelValue !== time || forceUpdated) { this.modelValue = time; this.propagateChange(time); } From e8dbe562191da733e29530dc1f452a41f4535776 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 23 Jan 2025 17:01:21 +0200 Subject: [PATCH 24/30] UI: Fixed advanced settings style in Save ts rule node config --- .../rule-node/action/timeseries-config.component.html | 3 ++- .../rule-node/common/time-unit-input.component.html | 4 ++-- .../components/rule-node/common/time-unit-input.component.ts | 4 ++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html index eb2811c8f8..e803d7443d 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html @@ -57,7 +57,7 @@ > } -
+
rule-node-config.advanced-settings @@ -71,6 +71,7 @@ - + {{ requiredText }} @@ -33,7 +33,7 @@ {{ maxErrorText }} - + rule-node-config.units @for (timeUnit of timeUnits; track timeUnit) { diff --git a/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts index 08d501477c..11358ae6eb 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts +++ b/ui-ngx/src/app/modules/home/components/rule-node/common/time-unit-input.component.ts @@ -30,6 +30,7 @@ import { isDefinedAndNotNull, isNumeric } from '@core/utils'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { coerceBoolean, coerceNumber } from '@shared/decorators/coercion'; import { DAY, HOUR, MINUTE, SECOND } from '@shared/models/time/time.models'; +import { SubscriptSizing } from '@angular/material/form-field'; interface TimeUnitInputModel { time: number; @@ -75,6 +76,9 @@ export class TimeUnitInputComponent implements ControlValueAccessor, Validator, @Input() maxErrorText: string; + @Input() + subscriptSizing: SubscriptSizing = 'fixed'; + timeUnits = Object.values(TimeUnit).filter(item => item !== TimeUnit.MILLISECONDS) as TimeUnit[]; timeUnitTranslations = timeUnitTranslations; From 2aeacf80c54f78e2793ce0b2e4066f8d045e1d88 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 31 Jan 2025 13:26:52 +0200 Subject: [PATCH 25/30] Clean up upgrade for next version --- .../main/data/upgrade/basic/schema_update.sql | 195 ------------------ .../install/ThingsboardInstallService.java | 19 +- .../service/install/InstallScripts.java | 20 -- .../update/DefaultDataUpdateService.java | 4 - 4 files changed, 3 insertions(+), 235 deletions(-) diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index 64bbbaca05..6b87dc6dde 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -14,198 +14,3 @@ -- limitations under the License. -- -ALTER TABLE user_credentials ADD COLUMN IF NOT EXISTS last_login_ts BIGINT; -UPDATE user_credentials c SET last_login_ts = (SELECT (additional_info::json ->> 'lastLoginTs')::bigint FROM tb_user u WHERE u.id = c.user_id) - WHERE last_login_ts IS NULL; - -ALTER TABLE user_credentials ADD COLUMN IF NOT EXISTS failed_login_attempts INT; -UPDATE user_credentials c SET failed_login_attempts = (SELECT (additional_info::json ->> 'failedLoginAttempts')::int FROM tb_user u WHERE u.id = c.user_id) - WHERE failed_login_attempts IS NULL; - -UPDATE tb_user SET additional_info = (additional_info::jsonb - 'lastLoginTs' - 'failedLoginAttempts' - 'userCredentialsEnabled')::text - WHERE additional_info IS NOT NULL AND additional_info != 'null' AND jsonb_typeof(additional_info::jsonb) = 'object'; - --- UPDATE RULE NODE DEBUG MODE TO DEBUG STRATEGY START - -ALTER TABLE rule_node ADD COLUMN IF NOT EXISTS debug_settings varchar(1024) DEFAULT null; -DO -$$ - BEGIN - IF EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'rule_node' AND column_name = 'debug_mode') - THEN - UPDATE rule_node SET debug_settings = '{"failuresEnabled": true, "allEnabledUntil": ' || cast((extract(epoch from now()) + 900) * 1000 as bigint) || '}' WHERE debug_mode = true; -- 15 minutes according to thingsboard.yml default settings. - ALTER TABLE rule_node DROP COLUMN debug_mode; - END IF; - END -$$; - --- UPDATE RULE NODE DEBUG MODE TO DEBUG STRATEGY END - - --- CREATE MOBILE APP BUNDLES FROM EXISTING APPS - -CREATE TABLE IF NOT EXISTS mobile_app_bundle ( - id uuid NOT NULL CONSTRAINT mobile_app_bundle_pkey PRIMARY KEY, - created_time bigint NOT NULL, - tenant_id uuid, - title varchar(255), - description varchar(1024), - android_app_id uuid UNIQUE, - ios_app_id uuid UNIQUE, - layout_config varchar(16384), - oauth2_enabled boolean, - CONSTRAINT fk_android_app_id FOREIGN KEY (android_app_id) REFERENCES mobile_app(id) ON DELETE SET NULL, - CONSTRAINT fk_ios_app_id FOREIGN KEY (ios_app_id) REFERENCES mobile_app(id) ON DELETE SET NULL -); -CREATE INDEX IF NOT EXISTS mobile_app_bundle_tenant_id ON mobile_app_bundle(tenant_id); - -ALTER TABLE mobile_app ADD COLUMN IF NOT EXISTS platform_type varchar(32), - ADD COLUMN IF NOT EXISTS status varchar(32), - ADD COLUMN IF NOT EXISTS version_info varchar(100000), - ADD COLUMN IF NOT EXISTS store_info varchar(16384), - DROP CONSTRAINT IF EXISTS mobile_app_pkg_name_key, - DROP CONSTRAINT IF EXISTS mobile_app_unq_key; - --- rename mobile_app_oauth2_client to mobile_app_bundle_oauth2_client -DO -$$ - BEGIN - -- in case of running the upgrade script a second time - IF EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name = 'mobile_app_oauth2_client') THEN - ALTER TABLE mobile_app_oauth2_client RENAME TO mobile_app_bundle_oauth2_client; - ALTER TABLE mobile_app_bundle_oauth2_client DROP CONSTRAINT IF EXISTS fk_domain; - ALTER TABLE mobile_app_bundle_oauth2_client RENAME COLUMN mobile_app_id TO mobile_app_bundle_id; - END IF; - END; -$$; - - -CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; - --- duplicate each mobile app and create mobile app bundle for the pair of android and ios app -DO -$$ - DECLARE - generatedBundleId uuid; - iosAppId uuid; - mobileAppRecord RECORD; - BEGIN - -- in case of running the upgrade script a second time - IF EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name = 'mobile_app' and column_name = 'oauth2_enabled') THEN - UPDATE mobile_app SET platform_type = 'ANDROID' WHERE platform_type IS NULL; - UPDATE mobile_app SET status = 'DRAFT' WHERE mobile_app.status IS NULL; - FOR mobileAppRecord IN SELECT * FROM mobile_app - LOOP - -- duplicate app for iOS platform type - iosAppId := uuid_generate_v4(); - INSERT INTO mobile_app(id, created_time, tenant_id, pkg_name, app_secret, platform_type, status) - VALUES (iosAppId, mobileAppRecord.created_time, mobileAppRecord.tenant_id, mobileAppRecord.pkg_name, mobileAppRecord.app_secret, 'IOS', mobileAppRecord.status) - ON CONFLICT DO NOTHING; - -- create bundle for android and iOS app - generatedBundleId := uuid_generate_v4(); - INSERT INTO mobile_app_bundle(id, created_time, tenant_id, title, android_app_id, ios_app_id, oauth2_enabled) - VALUES (generatedBundleId, mobileAppRecord.created_time, mobileAppRecord.tenant_id, - mobileAppRecord.pkg_name || ' (autogenerated)', mobileAppRecord.id, iosAppId, mobileAppRecord.oauth2_enabled) - ON CONFLICT DO NOTHING; - UPDATE mobile_app_bundle_oauth2_client SET mobile_app_bundle_id = generatedBundleId WHERE mobile_app_bundle_id = mobileAppRecord.id; - END LOOP; - END IF; - IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'fk_mobile_app_bundle_oauth2_client_bundle_id') THEN - ALTER TABLE mobile_app_bundle_oauth2_client ADD CONSTRAINT fk_mobile_app_bundle_oauth2_client_bundle_id - FOREIGN KEY (mobile_app_bundle_id) REFERENCES mobile_app_bundle(id) ON DELETE CASCADE; - END IF; - ALTER TABLE mobile_app DROP COLUMN IF EXISTS oauth2_enabled; - IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'mobile_app_pkg_name_platform_unq_key') THEN - ALTER TABLE mobile_app ADD CONSTRAINT mobile_app_pkg_name_platform_unq_key UNIQUE (pkg_name, platform_type); - END IF; - END; -$$; - -ALTER TABLE IF EXISTS mobile_app_settings RENAME TO qr_code_settings; -ALTER TABLE qr_code_settings ADD COLUMN IF NOT EXISTS mobile_app_bundle_id uuid, - ADD COLUMN IF NOT EXISTS android_enabled boolean, - ADD COLUMN IF NOT EXISTS ios_enabled boolean; - --- migrate mobile apps from qr code settings to mobile_app, create mobile app bundle for the pair of apps -DO -$$ - DECLARE - androidPkgName varchar; - iosPkgName varchar; - androidAppId uuid; - iosAppId uuid; - generatedBundleId uuid; - qrCodeRecord RECORD; - BEGIN - -- in case of running the upgrade script a second time - IF EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name = 'qr_code_settings' AND column_name = 'android_config') THEN - FOR qrCodeRecord IN SELECT * FROM qr_code_settings - LOOP - generatedBundleId := NULL; - -- migrate android config - IF (qrCodeRecord.android_config::jsonb ->> 'appPackage' IS NOT NULL) THEN - androidPkgName := qrCodeRecord.android_config::jsonb ->> 'appPackage'; - SELECT id into androidAppId FROM mobile_app WHERE pkg_name = androidPkgName AND platform_type = 'ANDROID'; - IF androidAppId IS NULL THEN - androidAppId := uuid_generate_v4(); - INSERT INTO mobile_app(id, created_time, tenant_id, pkg_name, platform_type, status, store_info) - VALUES (androidAppId, (extract(epoch from now()) * 1000), qrCodeRecord.tenant_id, - androidPkgName, 'ANDROID', 'DRAFT', qrCodeRecord.android_config::jsonb - 'appPackage' - 'enabled'); - generatedBundleId := uuid_generate_v4(); - INSERT INTO mobile_app_bundle(id, created_time, tenant_id, title, android_app_id) - VALUES (generatedBundleId, (extract(epoch from now()) * 1000), qrCodeRecord.tenant_id, androidPkgName || ' (autogenerated)', androidAppId); - UPDATE qr_code_settings SET mobile_app_bundle_id = generatedBundleId; - ELSE - UPDATE mobile_app SET store_info = qrCodeRecord.android_config::jsonb - 'appPackage' - 'enabled' WHERE id = androidAppId; - UPDATE qr_code_settings SET mobile_app_bundle_id = (SELECT id FROM mobile_app_bundle WHERE mobile_app_bundle.android_app_id = androidAppId); - END IF; - END IF; - UPDATE qr_code_settings SET android_enabled = (qrCodeRecord.android_config::jsonb ->> 'enabled')::boolean WHERE id = qrCodeRecord.id; - - -- migrate ios config - IF (qrCodeRecord.ios_config::jsonb ->> 'appId' IS NOT NULL) THEN - iosPkgName := substring(qrCodeRecord.ios_config::jsonb ->> 'appId', strpos(qrCodeRecord.ios_config::jsonb ->> 'appId', '.') + 1); - SELECT id INTO iosAppId FROM mobile_app WHERE pkg_name = iosPkgName AND platform_type = 'IOS'; - IF iosAppId IS NULL THEN - iosAppId := uuid_generate_v4(); - INSERT INTO mobile_app(id, created_time, tenant_id, pkg_name, platform_type, status, store_info) - VALUES (iosAppId, (extract(epoch from now()) * 1000), qrCodeRecord.tenant_id, - iosPkgName, 'IOS', 'DRAFT', qrCodeRecord.ios_config::jsonb - 'enabled'); - IF generatedBundleId IS NULL THEN - generatedBundleId := uuid_generate_v4(); - INSERT INTO mobile_app_bundle(id, created_time, tenant_id, title, ios_app_id) - VALUES (generatedBundleId, (extract(epoch from now()) * 1000), qrCodeRecord.tenant_id, iosPkgName || ' (autogenerated)', iosAppId); - UPDATE qr_code_settings SET mobile_app_bundle_id = generatedBundleId; - ELSE - UPDATE mobile_app_bundle SET ios_app_id = iosAppId WHERE id = generatedBundleId; - END IF; - ELSE - UPDATE qr_code_settings SET mobile_app_bundle_id = (SELECT id FROM mobile_app_bundle WHERE mobile_app_bundle.ios_app_id = iosAppId); - UPDATE mobile_app SET store_info = qrCodeRecord.ios_config::jsonb - 'enabled' WHERE id = iosAppId; - END IF; - END IF; - UPDATE qr_code_settings SET ios_enabled = (qrCodeRecord.ios_config::jsonb -> 'enabled')::boolean WHERE id = qrCodeRecord.id; - END LOOP; - ALTER TABLE qr_code_settings RENAME CONSTRAINT mobile_app_settings_tenant_id_unq_key TO qr_code_settings_tenant_id_unq_key; - ALTER TABLE qr_code_settings RENAME CONSTRAINT mobile_app_settings_pkey TO qr_code_settings_pkey; - END IF; - ALTER TABLE qr_code_settings DROP COLUMN IF EXISTS android_config, DROP COLUMN IF EXISTS ios_config; - END; -$$; - --- update constraint name -DO -$$ - BEGIN - ALTER TABLE domain DROP CONSTRAINT IF EXISTS domain_unq_key; - IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'domain_name_key') THEN - ALTER TABLE domain ADD CONSTRAINT domain_name_key UNIQUE (name); - END IF; - END; -$$; - --- UPDATE RESOURCE JS_MODULE SUB TYPE START - -UPDATE resource SET resource_sub_type = 'EXTENSION' WHERE resource_type = 'JS_MODULE' AND resource_sub_type IS NULL; - --- UPDATE RESOURCE JS_MODULE SUB TYPE END diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 622c0ab2dc..ea4c059222 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -35,8 +35,6 @@ import org.thingsboard.server.service.install.migrate.TsLatestMigrateService; import org.thingsboard.server.service.install.update.CacheCleanupService; import org.thingsboard.server.service.install.update.DataUpdateService; -import static org.thingsboard.server.service.install.update.DefaultDataUpdateService.getEnv; - @Service @Profile("install") @Slf4j @@ -99,8 +97,6 @@ public class ThingsboardInstallService { if ("cassandra-latest-to-postgres".equals(upgradeFromVersion)) { log.info("Migrating ThingsBoard latest timeseries data from cassandra to SQL database ..."); latestMigrateService.migrate(); - } else if (upgradeFromVersion.equals("3.9.0-resources")) { - installScripts.updateResourcesUsage(); } else { // TODO DON'T FORGET to update SUPPORTED_VERSIONS_FROM in DefaultDatabaseSchemaSettingsService databaseSchemaVersionService.validateSchemaSettings(); @@ -118,25 +114,16 @@ public class ThingsboardInstallService { entityDatabaseSchemaService.createOrUpdateDeviceInfoView(persistToTelemetry); // Creates missing indexes. entityDatabaseSchemaService.createDatabaseIndexes(); - // Runs upgrade scripts that are not possible in plain SQL. + // TODO: cleanup update code after each release - if (!getEnv("SKIP_RESOURCES_USAGE_MIGRATION", false)) { - installScripts.setUpdateResourcesUsage(true); - } else { - log.info("Skipping resources usage migration. Run the upgrade with fromVersion as '3.9.0-resources' to migrate"); - } - if (installScripts.isUpdateResourcesUsage()) { - installScripts.updateResourcesUsage(); - } + + // Runs upgrade scripts that are not possible in plain SQL. dataUpdateService.updateData(); log.info("Updating system data..."); dataUpdateService.upgradeRuleNodes(); systemDataLoaderService.loadSystemWidgets(); installScripts.loadSystemLwm2mResources(); installScripts.loadSystemImagesAndResources(); - if (installScripts.isUpdateImages()) { - installScripts.updateImages(); - } databaseSchemaVersionService.updateSchemaVersion(); } log.info("Upgrade finished successfully!"); diff --git a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java index a6ccb85e11..b230be0d57 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java +++ b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java @@ -17,8 +17,6 @@ package org.thingsboard.server.service.install; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import lombok.Getter; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -120,11 +118,6 @@ public class InstallScripts { @Autowired private ResourcesUpdater resourcesUpdater; - @Getter @Setter - private boolean updateImages = false; - - @Getter @Setter - private boolean updateResourcesUsage = false; @Autowired private ImageService imageService; @@ -395,14 +388,6 @@ public class InstallScripts { } } - public void updateImages() { - resourcesUpdater.updateWidgetsBundlesImages(); - resourcesUpdater.updateWidgetTypesImages(); - resourcesUpdater.updateDashboardsImages(); - resourcesUpdater.updateDeviceProfilesImages(); - resourcesUpdater.updateAssetProfilesImages(); - } - public void loadSystemImagesAndResources() { log.info("Loading system images and resources..."); Stream dashboardsFiles = Stream.concat(listDir(Paths.get(getDataDir(), JSON_DIR, DEMO_DIR, DASHBOARDS_DIR)), @@ -512,11 +497,6 @@ public class InstallScripts { } } - public void updateResourcesUsage() { - resourcesUpdater.updateDashboardsResources(); - resourcesUpdater.updateWidgetsResources(); - } - private void loadSystemResources(Path dir, ResourceType resourceType, ResourceSubType resourceSubType) { listDir(dir).forEach(resourceFile -> { String resourceKey = resourceFile.getFileName().toString(); diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index e8622fae37..cea938bce9 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -34,7 +34,6 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.sql.JpaExecutorService; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.component.RuleNodeClassInfo; -import org.thingsboard.server.service.install.InstallScripts; import org.thingsboard.server.utils.TbNodeUpgradeUtils; import java.util.ArrayList; @@ -58,9 +57,6 @@ public class DefaultDataUpdateService implements DataUpdateService { @Autowired JpaExecutorService jpaExecutorService; - @Autowired - private InstallScripts installScripts; - @Override public void updateData() throws Exception { log.info("Updating data ..."); From 82a823cac8cc77e4343f474ad255e253ed574064 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Wed, 5 Feb 2025 12:49:02 +0200 Subject: [PATCH 26/30] Add swagger endpoint for refresh jwt token --- .../server/config/SwaggerConfiguration.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/config/SwaggerConfiguration.java b/application/src/main/java/org/thingsboard/server/config/SwaggerConfiguration.java index c32533bd04..0f3f529cb7 100644 --- a/application/src/main/java/org/thingsboard/server/config/SwaggerConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/SwaggerConfiguration.java @@ -84,6 +84,7 @@ import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; public class SwaggerConfiguration { public static final String LOGIN_ENDPOINT = "/api/auth/login"; + public static final String REFRESH_TOKEN_ENDPOINT = "/api/auth/token"; private static final ApiResponses loginResponses = loginResponses(); private static final ApiResponses defaultErrorResponses = defaultErrorResponses(false); @@ -150,6 +151,7 @@ public class SwaggerConfiguration { .info(info); addDefaultSchemas(openApi); addLoginOperation(openApi); + addRefreshTokenOperation(openApi); return openApi; } @@ -210,6 +212,29 @@ public class SwaggerConfiguration { openAPI.path(LOGIN_ENDPOINT, pathItem); } + private void addRefreshTokenOperation(OpenAPI openAPI) { + var operation = new Operation(); + operation.summary("Refresh user JWT token data"); + operation.description(""" + Method to refresh JWT token. Provide a valid refresh token to get a new JWT token. + + The response contains a new token that can be used for authorization. + + `X-Authorization: Bearer $JWT_TOKEN_VALUE`"""); + + var requestBody = new RequestBody().description("Refresh token request") + .content(new Content().addMediaType(APPLICATION_JSON_VALUE, + new MediaType().schema(new Schema().addProperty("refreshToken", new Schema<>().type("string"))))); + + operation.requestBody(requestBody); + + operation.responses(loginResponses); + + operation.addTagsItem("login-endpoint"); + var pathItem = new PathItem().post(operation); + openAPI.path(REFRESH_TOKEN_ENDPOINT, pathItem); + } + @Bean public GroupedOpenApi groupedApi(SpringDocParameterNameDiscoverer localSpringDocParameterNameDiscoverer) { return GroupedOpenApi.builder() From 708c25d1ec944b008fdfd1513912ed7d9c08d2ae Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Thu, 6 Feb 2025 11:09:23 +0200 Subject: [PATCH 27/30] Save time series strategies: naming improvement --- .../rule/engine/telemetry/TbMsgTimeseriesNode.java | 2 +- .../engine/telemetry/TbMsgTimeseriesNodeConfiguration.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 61efffc79c..b98b1d205e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -182,7 +182,7 @@ public class TbMsgTimeseriesNode implements TbNode { return TimeseriesSaveRequest.Strategy.WS_ONLY; } if (persistenceSettings instanceof Deduplicate deduplicate) { - boolean isFirstMsgInInterval = deduplicate.getDeduplicateStrategy().shouldPersist(ts, originatorUuid); + boolean isFirstMsgInInterval = deduplicate.getPersistenceStrategy().shouldPersist(ts, originatorUuid); return isFirstMsgInInterval ? TimeseriesSaveRequest.Strategy.SAVE_ALL : TimeseriesSaveRequest.Strategy.SKIP_ALL; } if (persistenceSettings instanceof Advanced advanced) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java index fe22e42710..a1386fee49 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNodeConfiguration.java @@ -73,12 +73,12 @@ public class TbMsgTimeseriesNodeConfiguration implements NodeConfiguration Date: Thu, 6 Feb 2025 15:35:09 +0200 Subject: [PATCH 28/30] Resolve gateway dashboard branch automatically --- .../dashboard/DashboardSyncService.java | 8 ++++- .../DefaultDatabaseSchemaSettingsService.java | 12 +++---- .../server/service/install/ProjectInfo.java | 36 +++++++++++++++++++ .../src/main/resources/thingsboard.yml | 2 +- .../server/service/sync/vc/GitRepository.java | 2 +- 5 files changed, 50 insertions(+), 10 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/install/ProjectInfo.java diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DashboardSyncService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DashboardSyncService.java index 21eec71388..2adb6822dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DashboardSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DashboardSyncService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.entitiy.dashboard; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; @@ -30,6 +31,7 @@ import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.install.ProjectInfo; import org.thingsboard.server.service.sync.GitSyncService; import org.thingsboard.server.service.sync.vc.GitRepository.FileType; import org.thingsboard.server.service.sync.vc.GitRepository.RepoFile; @@ -51,10 +53,11 @@ public class DashboardSyncService { private final ImageService imageService; private final WidgetsBundleService widgetsBundleService; private final PartitionService partitionService; + private final ProjectInfo projectInfo; @Value("${transport.gateway.dashboard.sync.repository_url:}") private String repoUrl; - @Value("${transport.gateway.dashboard.sync.branch:main}") + @Value("${transport.gateway.dashboard.sync.branch:}") private String branch; @Value("${transport.gateway.dashboard.sync.fetch_frequency:24}") private int fetchFrequencyHours; @@ -64,6 +67,9 @@ public class DashboardSyncService { @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) public void init() throws Exception { + if (StringUtils.isBlank(branch)) { + branch = "release/" + projectInfo.getProjectVersion(); + } gitSyncService.registerSync(REPO_KEY, repoUrl, branch, TimeUnit.HOURS.toMillis(fetchFrequencyHours), this::update); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java index f1d9259e91..6fa8210125 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java @@ -19,7 +19,6 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.info.BuildProperties; import org.springframework.context.annotation.Profile; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; @@ -33,12 +32,11 @@ import java.util.List; @RequiredArgsConstructor public class DefaultDatabaseSchemaSettingsService implements DatabaseSchemaSettingsService { - private static final String CURRENT_PRODUCT = "CE"; // This list should include all versions which are compatible for the upgrade. // The compatibility cycle usually breaks when we have some scripts written in Java that may not work after new release. private static final List SUPPORTED_VERSIONS_FOR_UPGRADE = List.of("3.8.0", "3.8.1"); - private final BuildProperties buildProperties; + private final ProjectInfo projectInfo; private final JdbcTemplate jdbcTemplate; @Value("${install.upgrade.from_version:}") @@ -60,8 +58,8 @@ public class DefaultDatabaseSchemaSettingsService implements DatabaseSchemaSetti } String product = getProductFromDb(); - if (!CURRENT_PRODUCT.equals(product)) { - onSchemaSettingsError(String.format("Upgrade failed: can't upgrade ThingsBoard %s database using ThingsBoard %s.", product, CURRENT_PRODUCT)); + if (!projectInfo.getProductType().equals(product)) { + onSchemaSettingsError(String.format("Upgrade failed: can't upgrade ThingsBoard %s database using ThingsBoard %s.", product, projectInfo.getProductType())); } if (dbSchemaVersion.equals(getPackageSchemaVersion())) { @@ -87,7 +85,7 @@ public class DefaultDatabaseSchemaSettingsService implements DatabaseSchemaSetti public void createSchemaSettings() { Long schemaVersion = getSchemaVersionFromDb(); if (schemaVersion == null) { - jdbcTemplate.execute("INSERT INTO tb_schema_settings (schema_version, product) VALUES (" + getPackageSchemaVersionForDb() + ", '" + CURRENT_PRODUCT + "')"); + jdbcTemplate.execute("INSERT INTO tb_schema_settings (schema_version, product) VALUES (" + getPackageSchemaVersionForDb() + ", '" + projectInfo.getProductType() + "')"); } } @@ -99,7 +97,7 @@ public class DefaultDatabaseSchemaSettingsService implements DatabaseSchemaSetti @Override public String getPackageSchemaVersion() { if (packageSchemaVersion == null) { - packageSchemaVersion = buildProperties.getVersion().replaceAll("[^\\d.]", ""); + packageSchemaVersion = projectInfo.getProjectVersion(); } return packageSchemaVersion; } diff --git a/application/src/main/java/org/thingsboard/server/service/install/ProjectInfo.java b/application/src/main/java/org/thingsboard/server/service/install/ProjectInfo.java new file mode 100644 index 0000000000..eddd4c8ccf --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/ProjectInfo.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +import lombok.RequiredArgsConstructor; +import org.springframework.boot.info.BuildProperties; +import org.springframework.stereotype.Component; + +@Component +@RequiredArgsConstructor +public class ProjectInfo { + + private final BuildProperties buildProperties; + + public String getProjectVersion() { + return buildProperties.getVersion().replaceAll("[^\\d.]", ""); + } + + public String getProductType() { + return "CE"; + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index f29cc55197..368367bbab 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1271,7 +1271,7 @@ transport: # URL of gateways dashboard repository repository_url: "${TB_GATEWAY_DASHBOARD_SYNC_REPOSITORY_URL:https://github.com/thingsboard/gateway-management-extensions-dist.git}" # Branch of gateways dashboard repository to work with - branch: "${TB_GATEWAY_DASHBOARD_SYNC_BRANCH:main}" + branch: "${TB_GATEWAY_DASHBOARD_SYNC_BRANCH:}" # Fetch frequency in hours for gateways dashboard repository fetch_frequency: "${TB_GATEWAY_DASHBOARD_SYNC_FETCH_FREQUENCY:24}" diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 4841f05471..365fe23079 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -450,7 +450,7 @@ public class GitRepository { } ObjectId result = git.getRepository().resolve(rev); if (result == null) { - throw new IllegalArgumentException("Failed to parse git revision string: \"" + rev + "\""); + throw new IllegalArgumentException("Failed to resolve '" + rev + "'"); } return result; } From 908be4e1631c39e0aeac9f2498c05e2560228d57 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 6 Feb 2025 16:51:48 +0200 Subject: [PATCH 29/30] Update supported upgrade versions, cleanup --- .../DefaultDatabaseSchemaSettingsService.java | 30 ++----------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java index f1d9259e91..b414bfb13a 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java @@ -17,8 +17,6 @@ package org.thingsboard.server.service.install; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.info.BuildProperties; import org.springframework.context.annotation.Profile; import org.springframework.jdbc.core.JdbcTemplate; @@ -36,24 +34,16 @@ public class DefaultDatabaseSchemaSettingsService implements DatabaseSchemaSetti private static final String CURRENT_PRODUCT = "CE"; // This list should include all versions which are compatible for the upgrade. // The compatibility cycle usually breaks when we have some scripts written in Java that may not work after new release. - private static final List SUPPORTED_VERSIONS_FOR_UPGRADE = List.of("3.8.0", "3.8.1"); + private static final List SUPPORTED_VERSIONS_FOR_UPGRADE = List.of("3.9.0"); private final BuildProperties buildProperties; private final JdbcTemplate jdbcTemplate; - @Value("${install.upgrade.from_version:}") - private String upgradeFromVersion; - private String packageSchemaVersion; private String schemaVersionFromDb; @Override public void validateSchemaSettings() { - //TODO: remove after release (3.9.0) - createProductIfNotExists(); - - String dbSchemaVersion = getDbSchemaVersion(); - if (DefaultDataUpdateService.getEnv("SKIP_SCHEMA_VERSION_CHECK", false)) { log.info("Skipped DB schema version check due to SKIP_SCHEMA_VERSION_CHECK set to 'true'."); return; @@ -64,6 +54,7 @@ public class DefaultDatabaseSchemaSettingsService implements DatabaseSchemaSetti onSchemaSettingsError(String.format("Upgrade failed: can't upgrade ThingsBoard %s database using ThingsBoard %s.", product, CURRENT_PRODUCT)); } + String dbSchemaVersion = getDbSchemaVersion(); if (dbSchemaVersion.equals(getPackageSchemaVersion())) { onSchemaSettingsError("Upgrade failed: database already upgraded to current version. You can set SKIP_SCHEMA_VERSION_CHECK to 'true' if force re-upgrade needed."); } @@ -75,14 +66,6 @@ public class DefaultDatabaseSchemaSettingsService implements DatabaseSchemaSetti } } - @Deprecated(forRemoval = true, since = "3.9.0") - private void createProductIfNotExists() { - boolean isCommunityEdition = jdbcTemplate.queryForList( - "SELECT 1 FROM information_schema.tables WHERE table_name = 'integration'", Integer.class).isEmpty(); - String product = isCommunityEdition ? "CE" : "PE"; - jdbcTemplate.execute("ALTER TABLE tb_schema_settings ADD COLUMN IF NOT EXISTS product varchar(2) DEFAULT '" + product + "'"); - } - @Override public void createSchemaSettings() { Long schemaVersion = getSchemaVersionFromDb(); @@ -107,15 +90,6 @@ public class DefaultDatabaseSchemaSettingsService implements DatabaseSchemaSetti @Override public String getDbSchemaVersion() { if (schemaVersionFromDb == null) { - if (StringUtils.isNotBlank(upgradeFromVersion)) { - /* - * TODO - Remove after the release of 3.9.0: - * This a temporary workaround due to the issue that schema version in the - * tb_schema_settings was set as 3.6.4 during the install of 3.8.1. - * */ - schemaVersionFromDb = upgradeFromVersion; - return schemaVersionFromDb; - } Long version = getSchemaVersionFromDb(); if (version == null) { onSchemaSettingsError("Upgrade failed: the database schema version is missing."); From daf0b78ac5dc93e7faf5ca03a0741452975dcdf4 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 6 Feb 2025 17:07:13 +0200 Subject: [PATCH 30/30] Optimize imports --- .../service/install/DefaultDatabaseSchemaSettingsService.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java index 229e21a96d..3b21685d22 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultDatabaseSchemaSettingsService.java @@ -17,9 +17,6 @@ package org.thingsboard.server.service.install; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.info.BuildProperties; import org.springframework.context.annotation.Profile; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service;