From 264775ddb43da8aa6816f2bf9aee65325dc981b3 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 19 Mar 2025 20:51:27 +0100 Subject: [PATCH 01/79] MQTT client reconnect strategy exponential --- .../org/thingsboard/mqtt/MqttClientImpl.java | 5 +- .../thingsboard/mqtt/ReconnectStrategy.java | 21 +++++++ .../mqtt/ReconnectStrategyExponential.java | 58 +++++++++++++++++++ .../ReconnectStrategyExponentialTest.java | 43 ++++++++++++++ 4 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 netty-mqtt/src/main/java/org/thingsboard/mqtt/ReconnectStrategy.java create mode 100644 netty-mqtt/src/main/java/org/thingsboard/mqtt/ReconnectStrategyExponential.java create mode 100644 netty-mqtt/src/test/java/org/thingsboard/mqtt/ReconnectStrategyExponentialTest.java diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java index 47eae565dc..344886165e 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java @@ -80,6 +80,8 @@ final class MqttClientImpl implements MqttClient { private final MqttHandler defaultHandler; + private final ReconnectStrategy reconnectStrategy; + private EventLoopGroup eventLoop; private volatile Channel channel; @@ -110,6 +112,7 @@ final class MqttClientImpl implements MqttClient { this.clientConfig = clientConfig; this.defaultHandler = defaultHandler; this.handlerExecutor = handlerExecutor; + this.reconnectStrategy = new ReconnectStrategyExponential(getClientConfig().getReconnectDelay()); } /** @@ -191,7 +194,7 @@ final class MqttClientImpl implements MqttClient { if (reconnect) { this.reconnect = true; } - eventLoop.schedule((Runnable) () -> connect(host, port, reconnect), clientConfig.getReconnectDelay(), TimeUnit.SECONDS); + eventLoop.schedule((Runnable) () -> connect(host, port, reconnect), reconnectStrategy.getNextReconnectDelay(), TimeUnit.SECONDS); } } diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/ReconnectStrategy.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/ReconnectStrategy.java new file mode 100644 index 0000000000..f924b79ca5 --- /dev/null +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/ReconnectStrategy.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt; + +@FunctionalInterface +public interface ReconnectStrategy { + long getNextReconnectDelay(); +} diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/ReconnectStrategyExponential.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/ReconnectStrategyExponential.java new file mode 100644 index 0000000000..dbd11f91e1 --- /dev/null +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/ReconnectStrategyExponential.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.util.concurrent.TimeUnit; + +@Getter +@Slf4j +public class ReconnectStrategyExponential implements ReconnectStrategy { + + public static final int DEFAULT_RECONNECT_INTERVAL = 10; + final long reconnectIntervalMinSeconds; + final long reconnectIntervalMaxSeconds = 30; + long lastDisconnectNanoTime = 0; //isotonic time + int retryCount = 0; + + public ReconnectStrategyExponential(long reconnectIntervalMin) { + this.reconnectIntervalMinSeconds = calculateIntervalMin(reconnectIntervalMin); + } + + private long calculateIntervalMin(long reconnectIntervalMin) { + return Math.min((reconnectIntervalMin > 0 ? reconnectIntervalMin : DEFAULT_RECONNECT_INTERVAL), this.reconnectIntervalMaxSeconds); + } + + @Override + synchronized public long getNextReconnectDelay() { + final long currentNanoTime = getNanoTime(); + final long lastDisconnectIntervalNanos = currentNanoTime - lastDisconnectNanoTime; + lastDisconnectNanoTime = currentNanoTime; + if (TimeUnit.NANOSECONDS.toSeconds(lastDisconnectIntervalNanos) > reconnectIntervalMaxSeconds + reconnectIntervalMinSeconds) { + log.debug("Reset retry counter"); + retryCount = 0; + return reconnectIntervalMinSeconds; + } + return Math.min(reconnectIntervalMaxSeconds, reconnectIntervalMinSeconds + (1L << retryCount++)); + } + + long getNanoTime() { + return System.nanoTime(); + } + +} diff --git a/netty-mqtt/src/test/java/org/thingsboard/mqtt/ReconnectStrategyExponentialTest.java b/netty-mqtt/src/test/java/org/thingsboard/mqtt/ReconnectStrategyExponentialTest.java new file mode 100644 index 0000000000..828cfae730 --- /dev/null +++ b/netty-mqtt/src/test/java/org/thingsboard/mqtt/ReconnectStrategyExponentialTest.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; +import org.mockito.BDDMockito; +import org.mockito.Mockito; + +import java.util.concurrent.TimeUnit; + +@Slf4j +class ReconnectStrategyExponentialTest { + + @Test + public void exponentialReconnect() { + ReconnectStrategyExponential strategy = Mockito.spy(new ReconnectStrategyExponential(1)); + for (int i = 0; i < 10; i++) { + log.info("Disconnect [{}] Delay [{}]", i, strategy.getNextReconnectDelay()); + } + + final long coolDownPeriod = strategy.getReconnectIntervalMinSeconds() + strategy.getReconnectIntervalMaxSeconds() + 1; + + BDDMockito.willAnswer((x) -> System.nanoTime() + TimeUnit.SECONDS.toNanos(coolDownPeriod)).given(strategy).getNanoTime(); + log.info("After cooldown period [{}] seconds later...", coolDownPeriod); + for (int i = 0; i < 10; i++) { + log.info("Disconnect [{}] Delay [{}]", i, strategy.getNextReconnectDelay()); + } + } +} \ No newline at end of file From 83790fa0fb6a208ca872a7c9cb704d2e8dab80e6 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 20 Mar 2025 11:02:52 +0100 Subject: [PATCH 02/79] ReconnectStrategyExponential jitter, max added, refactored, tested --- .../mqtt/ReconnectStrategyExponential.java | 50 +++++++++--- .../ReconnectStrategyExponentialTest.java | 80 +++++++++++++++---- 2 files changed, 103 insertions(+), 27 deletions(-) diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/ReconnectStrategyExponential.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/ReconnectStrategyExponential.java index dbd11f91e1..e0b83c6ab7 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/ReconnectStrategyExponential.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/ReconnectStrategyExponential.java @@ -18,37 +18,61 @@ package org.thingsboard.mqtt; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; @Getter @Slf4j public class ReconnectStrategyExponential implements ReconnectStrategy { - public static final int DEFAULT_RECONNECT_INTERVAL = 10; - final long reconnectIntervalMinSeconds; - final long reconnectIntervalMaxSeconds = 30; - long lastDisconnectNanoTime = 0; //isotonic time - int retryCount = 0; + public static final int DEFAULT_RECONNECT_INTERVAL_SEC = 10; + public static final int MAX_RECONNECT_INTERVAL_SEC = 60; + public static final int EXP_MAX = 8; + public static final long JITTER_MAX = 1; + private final long reconnectIntervalMinSeconds; + private final long reconnectIntervalMaxSeconds; + private long lastDisconnectNanoTime = 0; //isotonic time + private long retryCount = 0; - public ReconnectStrategyExponential(long reconnectIntervalMin) { - this.reconnectIntervalMinSeconds = calculateIntervalMin(reconnectIntervalMin); + public ReconnectStrategyExponential(long reconnectIntervalMinSeconds) { + this.reconnectIntervalMaxSeconds = calculateIntervalMax(reconnectIntervalMinSeconds); + this.reconnectIntervalMinSeconds = calculateIntervalMin(reconnectIntervalMinSeconds); } - private long calculateIntervalMin(long reconnectIntervalMin) { - return Math.min((reconnectIntervalMin > 0 ? reconnectIntervalMin : DEFAULT_RECONNECT_INTERVAL), this.reconnectIntervalMaxSeconds); + long calculateIntervalMax(long reconnectIntervalMinSeconds) { + return reconnectIntervalMinSeconds > MAX_RECONNECT_INTERVAL_SEC ? reconnectIntervalMinSeconds : MAX_RECONNECT_INTERVAL_SEC; + } + + long calculateIntervalMin(long reconnectIntervalMinSeconds) { + return Math.min((reconnectIntervalMinSeconds > 0 ? reconnectIntervalMinSeconds : DEFAULT_RECONNECT_INTERVAL_SEC), this.reconnectIntervalMaxSeconds); } @Override synchronized public long getNextReconnectDelay() { final long currentNanoTime = getNanoTime(); - final long lastDisconnectIntervalNanos = currentNanoTime - lastDisconnectNanoTime; + final long coolDownSpentNanos = currentNanoTime - lastDisconnectNanoTime; lastDisconnectNanoTime = currentNanoTime; - if (TimeUnit.NANOSECONDS.toSeconds(lastDisconnectIntervalNanos) > reconnectIntervalMaxSeconds + reconnectIntervalMinSeconds) { - log.debug("Reset retry counter"); + if (isCooledDown(coolDownSpentNanos)) { retryCount = 0; return reconnectIntervalMinSeconds; } - return Math.min(reconnectIntervalMaxSeconds, reconnectIntervalMinSeconds + (1L << retryCount++)); + return calculateNextReconnectDelay() + calculateJitter(); + } + + long calculateJitter() { + return ThreadLocalRandom.current().nextInt() >= 0 ? JITTER_MAX : 0; + } + + long calculateNextReconnectDelay() { + return Math.min(reconnectIntervalMaxSeconds, reconnectIntervalMinSeconds + calculateExp(retryCount++)); + } + + long calculateExp(long e) { + return 1L << Math.min(e, EXP_MAX); + } + + boolean isCooledDown(long coolDownSpentNanos) { + return TimeUnit.NANOSECONDS.toSeconds(coolDownSpentNanos) > reconnectIntervalMaxSeconds + reconnectIntervalMinSeconds; } long getNanoTime() { diff --git a/netty-mqtt/src/test/java/org/thingsboard/mqtt/ReconnectStrategyExponentialTest.java b/netty-mqtt/src/test/java/org/thingsboard/mqtt/ReconnectStrategyExponentialTest.java index 828cfae730..085dcef3a6 100644 --- a/netty-mqtt/src/test/java/org/thingsboard/mqtt/ReconnectStrategyExponentialTest.java +++ b/netty-mqtt/src/test/java/org/thingsboard/mqtt/ReconnectStrategyExponentialTest.java @@ -16,28 +16,80 @@ package org.thingsboard.mqtt; import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.Test; -import org.mockito.BDDMockito; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mockito; +import org.mockito.stubbing.Answer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.data.Offset.offset; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.BDDMockito.willAnswer; +import static org.thingsboard.mqtt.ReconnectStrategyExponential.EXP_MAX; +import static org.thingsboard.mqtt.ReconnectStrategyExponential.JITTER_MAX; @Slf4j class ReconnectStrategyExponentialTest { - @Test - public void exponentialReconnect() { - ReconnectStrategyExponential strategy = Mockito.spy(new ReconnectStrategyExponential(1)); - for (int i = 0; i < 10; i++) { - log.info("Disconnect [{}] Delay [{}]", i, strategy.getNextReconnectDelay()); - } + @Execution(ExecutionMode.SAME_THREAD) // just for convenient log reading + @ParameterizedTest + @ValueSource(ints = {1, 0, 60}) + public void exponentialReconnectDelayTest(final int reconnectIntervalMinSeconds) { + final ReconnectStrategyExponential strategy = Mockito.spy(new ReconnectStrategyExponential(reconnectIntervalMinSeconds)); + log.info("=== Reconnect delay test for ReconnectStrategyExponential({}) : calculated min [{}] max [{}] ===", reconnectIntervalMinSeconds, strategy.getReconnectIntervalMinSeconds(), strategy.getReconnectIntervalMaxSeconds()); + final AtomicLong nanoTime = new AtomicLong(System.nanoTime()); + willAnswer((x) -> nanoTime.get()).given(strategy).getNanoTime(); + final LinkedBlockingDeque jittersCaptured = new LinkedBlockingDeque<>(); + final LinkedBlockingDeque expCaptured = new LinkedBlockingDeque<>(); - final long coolDownPeriod = strategy.getReconnectIntervalMinSeconds() + strategy.getReconnectIntervalMaxSeconds() + 1; + willAnswer(captureResult(jittersCaptured)).given(strategy).calculateJitter(); + willAnswer(captureResult(expCaptured)).given(strategy).calculateExp(anyLong()); - BDDMockito.willAnswer((x) -> System.nanoTime() + TimeUnit.SECONDS.toNanos(coolDownPeriod)).given(strategy).getNanoTime(); - log.info("After cooldown period [{}] seconds later...", coolDownPeriod); - for (int i = 0; i < 10; i++) { - log.info("Disconnect [{}] Delay [{}]", i, strategy.getNextReconnectDelay()); + for (int phase = 0; phase < 3; phase++) { + log.info("== Phase {} ==", phase); + long previousDelay = 0; + for (int i = 0; i < EXP_MAX + 4; i++) { + final long nextReconnectDelay = strategy.getNextReconnectDelay(); + nanoTime.addAndGet(TimeUnit.SECONDS.toNanos(nextReconnectDelay)); + log.info("Retry [{}] Delay [{}] : min [{}] exp [{}] jitter [{}]", strategy.getRetryCount(), nextReconnectDelay, strategy.getReconnectIntervalMinSeconds(), expCaptured.peekLast(), jittersCaptured.peekLast()); + assertThat(previousDelay).satisfiesAnyOf( + v -> assertThat(v).isLessThanOrEqualTo(nextReconnectDelay), + v -> assertThat(v).isCloseTo(nextReconnectDelay, offset(JITTER_MAX)) // Adjust tolerance as needed + ); + previousDelay = nextReconnectDelay; + } + log.info("Jitters captured: {}", drainAll(jittersCaptured)); + log.info("Exponents captured: {}", drainAll(expCaptured)); + assertThat(previousDelay).isCloseTo(strategy.getReconnectIntervalMaxSeconds(), offset(JITTER_MAX)); + + final long coolDownPeriodSec = strategy.getReconnectIntervalMinSeconds() + strategy.getReconnectIntervalMaxSeconds() + 1; + log.info("Cooling down for [{}] seconds ...", coolDownPeriodSec); + nanoTime.addAndGet(TimeUnit.SECONDS.toNanos(coolDownPeriodSec)); + assertThat(strategy.isCooledDown(TimeUnit.SECONDS.toNanos(coolDownPeriodSec))).as("cooled down").isTrue(); } } -} \ No newline at end of file + + private Answer captureResult(Collection collection) { + return invocation -> { + long result = (long) invocation.callRealMethod(); + collection.add(result); + return result; + }; + } + + private Collection drainAll(BlockingQueue jittersCaptured) { + Collection elements = new ArrayList<>(); + jittersCaptured.drainTo(elements); + return elements; + } + +} From de855e5236dea39f7b0967dfb3f24af775ffe562 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 11 Apr 2025 16:56:00 +0300 Subject: [PATCH 03/79] Minor improvements for EDQS --- .../java/org/thingsboard/server/edqs/util/VersionsStore.java | 2 +- .../server/queue/kafka/TbKafkaConsumerStatsService.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java index 798ac0603d..4a09a76e9a 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java @@ -36,7 +36,7 @@ public class VersionsStore { isNew.set(true); return version; } else { - log.info("[{}] Version {} is outdated, the latest is {}", key, version, prevVersion); + log.debug("[{}] Version {} is outdated, the latest is {}", key, version, prevVersion); return prevVersion; } }); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java index 9d1088e188..7a9c01b72f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java @@ -93,8 +93,7 @@ public class TbKafkaConsumerStatsService { log.info("[{}] Topic partitions with lag: [{}].", groupId, builder.toString()); } } catch (Exception e) { - log.warn("[{}] Failed to get consumer group stats. Reason - {}.", groupId, e.getMessage()); - log.trace("Detailed error: ", e); + log.warn("[{}] Failed to get consumer group stats", groupId, e); } } From f8bf512a0a44fee0e85eed611e9abc02c5793fb0 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 15 Apr 2025 11:51:45 +0300 Subject: [PATCH 04/79] Introduce PartitionedQueueResponseTemplate with consumer per partition; use it for EDQS requests processing --- .../src/main/resources/thingsboard.yml | 2 + .../server/queue/TbQueueProducer.java | 3 +- .../server/edqs/processor/EdqsProcessor.java | 46 ++--- .../server/edqs/state/EdqsStateService.java | 3 +- .../edqs/state/KafkaEdqsStateService.java | 14 +- .../edqs/state/LocalEdqsStateService.java | 8 +- .../common/DefaultTbQueueRequestTemplate.java | 1 - .../DefaultTbQueueResponseTemplate.java | 5 - .../PartitionedQueueResponseTemplate.java | 164 ++++++++++++++++++ .../consumer/MainQueueConsumerManager.java | 5 +- .../PartitionedQueueConsumerManager.java | 3 +- .../state/DefaultQueueStateService.java | 4 +- .../common/state/KafkaQueueStateService.java | 8 +- .../queue/common/state/QueueStateService.java | 14 +- .../server/queue/edqs/EdqsConfig.java | 2 + .../server/queue/edqs/EdqsExecutors.java | 70 ++++++++ .../server/queue/edqs/EdqsQueueFactory.java | 5 +- .../queue/edqs/InMemoryEdqsQueueFactory.java | 26 +-- .../queue/edqs/KafkaEdqsQueueFactory.java | 35 ++-- .../queue/kafka/TbKafkaProducerTemplate.java | 4 - .../queue/memory/InMemoryTbQueueProducer.java | 5 - .../DefaultTbQueueRequestTemplateTest.java | 19 +- edqs/src/main/resources/edqs.yml | 2 + 23 files changed, 350 insertions(+), 98 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/PartitionedQueueResponseTemplate.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsExecutors.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index abafd81e7c..6ad3386c4f 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1778,6 +1778,8 @@ queue: max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}" # Maximum timeout for requests to EDQS max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:20000}" + # Thread pool size for EDQS requests executor + request_executor_size: "${TB_EDQS_REQUEST_EXECUTOR_SIZE:50}" # Strings longer than this threshold will be compressed string_compression_length_threshold: "${TB_EDQS_STRING_COMPRESSION_LENGTH_THRESHOLD:512}" stats: diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java index 51fd735a9d..a374928b16 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueProducer.java @@ -19,11 +19,10 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; public interface TbQueueProducer { - void init(); - String getDefaultTopic(); void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback); void stop(); + } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java index 78d17ef368..7b30fbaaa5 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java @@ -18,7 +18,6 @@ package org.thingsboard.server.edqs.processor; import com.google.common.collect.Sets; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Getter; @@ -29,7 +28,6 @@ import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ExceptionUtil; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.EdqsEvent; @@ -54,7 +52,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.queue.TbQueueHandler; -import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.common.PartitionedQueueResponseTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; import org.thingsboard.server.queue.discovery.QueueKey; @@ -62,15 +60,14 @@ import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.edqs.EdqsComponent; import org.thingsboard.server.queue.edqs.EdqsConfig; import org.thingsboard.server.queue.edqs.EdqsConfig.EdqsPartitioningStrategy; +import org.thingsboard.server.queue.edqs.EdqsExecutors; import org.thingsboard.server.queue.edqs.EdqsQueueFactory; -import org.thingsboard.server.queue.util.AfterStartUp; +import java.util.List; import java.util.Objects; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -87,20 +84,16 @@ public class EdqsProcessor implements TbQueueHandler, private final EdqsConverter converter; private final EdqsRepository repository; private final EdqsConfig config; + private final EdqsExecutors edqsExecutors; private final EdqsPartitionService partitionService; private final ConfigurableApplicationContext applicationContext; private final EdqsStateService stateService; private PartitionedQueueConsumerManager> eventConsumer; - private TbQueueResponseTemplate, TbProtoQueueMsg> responseTemplate; - - private ExecutorService consumersExecutor; - private ExecutorService taskExecutor; - private ScheduledExecutorService scheduler; + private PartitionedQueueResponseTemplate, TbProtoQueueMsg> responseTemplate; private ListeningExecutorService requestExecutor; private final VersionsStore versionsStore = new VersionsStore(); - private final AtomicInteger counter = new AtomicInteger(); @Getter @@ -108,10 +101,6 @@ public class EdqsProcessor implements TbQueueHandler, @PostConstruct private void init() { - consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("edqs-consumer")); - taskExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "edqs-consumer-task-executor"); - scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-scheduler"); - requestExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(12, "edqs-requests")); errorHandler = error -> { if (error instanceof OutOfMemoryError) { log.error("OOM detected, shutting down"); @@ -120,6 +109,7 @@ public class EdqsProcessor implements TbQueueHandler, .execute(applicationContext::close); } }; + requestExecutor = edqsExecutors.getRequestExecutor(); eventConsumer = PartitionedQueueConsumerManager.>create() .queueKey(new QueueKey(ServiceType.EDQS, config.getEventsTopic())) @@ -141,19 +131,14 @@ public class EdqsProcessor implements TbQueueHandler, }) .consumerCreator((config, tpi) -> queueFactory.createEdqsEventsConsumer()) .queueAdmin(queueFactory.getEdqsQueueAdmin()) - .consumerExecutor(consumersExecutor) - .taskExecutor(taskExecutor) - .scheduler(scheduler) + .consumerExecutor(edqsExecutors.getConsumersExecutor()) + .taskExecutor(edqsExecutors.getConsumerTaskExecutor()) + .scheduler(edqsExecutors.getScheduler()) .uncaughtErrorHandler(errorHandler) .build(); - stateService.init(eventConsumer); - - responseTemplate = queueFactory.createEdqsResponseTemplate(); - } + responseTemplate = queueFactory.createEdqsResponseTemplate(this); - @AfterStartUp(order = 1) - public void start() { - responseTemplate.launch(this); + stateService.init(eventConsumer, List.of(responseTemplate.getRequestConsumer())); } @EventListener @@ -163,10 +148,8 @@ public class EdqsProcessor implements TbQueueHandler, } try { Set newPartitions = event.getNewPartitions().get(new QueueKey(ServiceType.EDQS)); - stateService.process(withTopic(newPartitions, config.getStateTopic())); - // eventsConsumer's partitions are updated by stateService - responseTemplate.subscribe(withTopic(newPartitions, config.getRequestsTopic())); // TODO: we subscribe to partitions before we are ready. implement consumer-per-partition version for request template + // partitions for event and request consumers are updated by stateService Set oldPartitions = event.getOldPartitions().get(new QueueKey(ServiceType.EDQS)); if (CollectionsUtil.isNotEmpty(oldPartitions)) { @@ -290,11 +273,6 @@ public class EdqsProcessor implements TbQueueHandler, eventConsumer.awaitStop(); responseTemplate.stop(); stateService.stop(); - - consumersExecutor.shutdownNow(); - taskExecutor.shutdownNow(); - scheduler.shutdownNow(); - requestExecutor.shutdownNow(); } } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java index ee7b058a8a..050fa74e6b 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java @@ -23,11 +23,12 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; +import java.util.List; import java.util.Set; public interface EdqsStateService { - void init(PartitionedQueueConsumerManager> eventConsumer); + void init(PartitionedQueueConsumerManager> eventConsumer, List> otherConsumers); void process(Set partitions); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java index 0efe6e7d3b..e8ede39d2c 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java @@ -37,12 +37,14 @@ import org.thingsboard.server.queue.common.state.KafkaQueueStateService; import org.thingsboard.server.queue.common.state.QueueStateService; import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsExecutors; import org.thingsboard.server.queue.edqs.KafkaEdqsComponent; import org.thingsboard.server.queue.edqs.KafkaEdqsQueueFactory; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -59,6 +61,7 @@ public class KafkaEdqsStateService implements EdqsStateService { private final EdqsConfig config; private final EdqsPartitionService partitionService; private final KafkaEdqsQueueFactory queueFactory; + private final EdqsExecutors edqsExecutors; @Autowired @Lazy private EdqsProcessor edqsProcessor; @@ -74,7 +77,7 @@ public class KafkaEdqsStateService implements EdqsStateService { private Boolean ready; @Override - public void init(PartitionedQueueConsumerManager> eventConsumer) { + public void init(PartitionedQueueConsumerManager> eventConsumer, List> otherConsumers) { TbKafkaAdmin queueAdmin = queueFactory.getEdqsQueueAdmin(); stateConsumer = PartitionedQueueConsumerManager.>create() .queueKey(new QueueKey(ServiceType.EDQS, config.getStateTopic())) @@ -96,9 +99,9 @@ public class KafkaEdqsStateService implements EdqsStateService { }) .consumerCreator((config, tpi) -> queueFactory.createEdqsStateConsumer()) .queueAdmin(queueAdmin) - .consumerExecutor(eventConsumer.getConsumerExecutor()) - .taskExecutor(eventConsumer.getTaskExecutor()) - .scheduler(eventConsumer.getScheduler()) + .consumerExecutor(edqsExecutors.getConsumersExecutor()) + .taskExecutor(edqsExecutors.getConsumerTaskExecutor()) + .scheduler(edqsExecutors.getScheduler()) .uncaughtErrorHandler(edqsProcessor.getErrorHandler()) .build(); @@ -141,7 +144,7 @@ public class KafkaEdqsStateService implements EdqsStateService { consumer.commit(); }) .consumerCreator(() -> eventsToBackupKafkaConsumer) - .consumerExecutor(eventConsumer.getConsumerExecutor()) + .consumerExecutor(edqsExecutors.getConsumersExecutor()) .threadPrefix("edqs-events-to-backup") .build(); @@ -153,6 +156,7 @@ public class KafkaEdqsStateService implements EdqsStateService { queueStateService = KafkaQueueStateService., TbProtoQueueMsg>builder() .eventConsumer(eventConsumer) .stateConsumer(stateConsumer) + .otherConsumers(otherConsumers) .eventsStartOffsetsProvider(() -> { // taking start offsets for events topics from the events-to-backup consumer group, // since eventConsumer doesn't use consumer group management and thus offset tracking diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java index cde21edfaf..c9923e6194 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java @@ -31,6 +31,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; import org.thingsboard.server.queue.edqs.InMemoryEdqsComponent; +import java.util.List; import java.util.Set; import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTopic; @@ -46,11 +47,13 @@ public class LocalEdqsStateService implements EdqsStateService { private EdqsProcessor processor; private PartitionedQueueConsumerManager> eventConsumer; + private List> otherConsumers; private Set partitions; @Override - public void init(PartitionedQueueConsumerManager> eventConsumer) { + public void init(PartitionedQueueConsumerManager> eventConsumer, List> otherConsumers) { this.eventConsumer = eventConsumer; + this.otherConsumers = otherConsumers; } @Override @@ -68,6 +71,9 @@ public class LocalEdqsStateService implements EdqsStateService { log.info("Restore completed"); } eventConsumer.update(withTopic(partitions, eventConsumer.getTopic())); + for (PartitionedQueueConsumerManager consumer : otherConsumers) { + consumer.update(withTopic(partitions, consumer.getTopic())); + } this.partitions = partitions; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 1beb505595..c6f76a5000 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -91,7 +91,6 @@ public class DefaultTbQueueRequestTemplate requestTemplate; private final TbQueueProducer responseTemplate; - private final ConcurrentMap pendingRequests; private final ExecutorService loopExecutor; private final ScheduledExecutorService timeoutExecutor; private final ExecutorService callbackExecutor; @@ -67,7 +64,6 @@ public class DefaultTbQueueResponseTemplate(); this.maxPendingRequests = maxPendingRequests; this.pollInterval = pollInterval; this.requestTimeout = requestTimeout; @@ -89,7 +85,6 @@ public class DefaultTbQueueResponseTemplate handler) { - this.responseTemplate.init(); loopExecutor.submit(() -> { while (!stopped) { try { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/PartitionedQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/PartitionedQueueResponseTemplate.java new file mode 100644 index 0000000000..d1e1be708f --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/PartitionedQueueResponseTemplate.java @@ -0,0 +1,164 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import lombok.Builder; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.stats.MessagesStats; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueHandler; +import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; + +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; + +@Slf4j +public class PartitionedQueueResponseTemplate extends AbstractTbQueueTemplate { + + @Getter + private final PartitionedQueueConsumerManager requestConsumer; + private final TbQueueProducer responseProducer; + + private final TbQueueHandler handler; + private final long pollInterval; + private final int maxPendingRequests; + private final long requestTimeout; + private final MessagesStats stats; + + private final ScheduledExecutorService scheduler; + private final ExecutorService callbackExecutor; + + private final AtomicInteger pendingRequestCount = new AtomicInteger(); + + @Builder + public PartitionedQueueResponseTemplate(String key, + TbQueueHandler handler, + String requestsTopic, + Function> consumerCreator, + TbQueueProducer responseProducer, + long pollInterval, + long requestTimeout, + int maxPendingRequests, + ExecutorService consumerExecutor, + ExecutorService callbackExecutor, + ExecutorService consumerTaskExecutor, + MessagesStats stats) { + this.scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor(key + "-queue-response-template-scheduler"); + this.callbackExecutor = callbackExecutor; + this.handler = handler; + this.requestConsumer = PartitionedQueueConsumerManager.create() + .queueKey(key + "-requests") + .topic(requestsTopic) + .pollInterval(pollInterval) + .msgPackProcessor((requests, consumer, config) -> processRequests(requests, consumer)) + .consumerCreator((config, tpi) -> consumerCreator.apply(tpi)) + .consumerExecutor(consumerExecutor) + .scheduler(scheduler) + .taskExecutor(consumerTaskExecutor) + .build(); + this.responseProducer = responseProducer; + this.pollInterval = pollInterval; + this.maxPendingRequests = maxPendingRequests; + this.requestTimeout = requestTimeout; + this.stats = stats; + } + + private void processRequests(List requests, TbQueueConsumer consumer) { + while (pendingRequestCount.get() >= maxPendingRequests) { + try { + Thread.sleep(pollInterval); + } catch (InterruptedException e) { + log.trace("Failed to wait until the server has capacity to handle new requests", e); + } + } + + requests.forEach(request -> { + long currentTime = System.currentTimeMillis(); + long expireTs = bytesToLong(request.getHeaders().get(EXPIRE_TS_HEADER)); + if (expireTs >= currentTime) { + byte[] requestIdHeader = request.getHeaders().get(REQUEST_ID_HEADER); + if (requestIdHeader == null) { + log.error("[{}] Missing requestId in header", request); + return; + } + byte[] responseTopicHeader = request.getHeaders().get(RESPONSE_TOPIC_HEADER); + if (responseTopicHeader == null) { + log.error("[{}] Missing response topic in header", request); + return; + } + UUID requestId = bytesToUuid(requestIdHeader); + String responseTopic = bytesToString(responseTopicHeader); + try { + pendingRequestCount.getAndIncrement(); + stats.incrementTotal(); + AsyncCallbackTemplate.withCallbackAndTimeout(handler.handle(request), + response -> { + pendingRequestCount.decrementAndGet(); + response.getHeaders().put(REQUEST_ID_HEADER, uuidToBytes(requestId)); + responseProducer.send(TopicPartitionInfo.builder().topic(responseTopic).build(), response, null); + stats.incrementSuccessful(); + }, + e -> { + pendingRequestCount.decrementAndGet(); + if (e.getCause() != null && e.getCause() instanceof TimeoutException) { + log.warn("[{}] Timeout to process the request: {}", requestId, request, e); + } else { + log.trace("[{}] Failed to process the request: {}", requestId, request, e); + } + stats.incrementFailed(); + }, + requestTimeout, + scheduler, + callbackExecutor); + } catch (Throwable e) { + pendingRequestCount.decrementAndGet(); + log.warn("[{}] Failed to process the request: {}", requestId, request, e); + stats.incrementFailed(); + } + } + }); + consumer.commit(); + } + + public void subscribe(Set partitions) { + requestConsumer.update(partitions); + } + + public void stop() { + if (requestConsumer != null) { + requestConsumer.stop(); + requestConsumer.awaitStop(); + } + if (responseProducer != null) { + responseProducer.stop(); + } + if (scheduler != null) { + scheduler.shutdownNow(); + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java index 2233855a37..86e0721dd8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java @@ -25,7 +25,6 @@ import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.UpdateConfigTask; import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.UpdatePartitionsTask; -import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import java.util.Collection; @@ -50,7 +49,7 @@ import java.util.function.Function; public class MainQueueConsumerManager { @Getter - protected final QueueKey queueKey; + protected final Object queueKey; @Getter protected C config; protected final MsgPackProcessor msgPackProcessor; @@ -72,7 +71,7 @@ public class MainQueueConsumerManager msgPackProcessor, BiFunction> consumerCreator, ExecutorService consumerExecutor, diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java index 1b19fcab0e..42d5b6bb18 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java @@ -26,7 +26,6 @@ import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.AddPartitionsTask; import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.DeletePartitionsTask; import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask.RemovePartitionsTask; -import org.thingsboard.server.queue.discovery.QueueKey; import java.util.Set; import java.util.concurrent.ExecutorService; @@ -44,7 +43,7 @@ public class PartitionedQueueConsumerManager extends MainQ private final String topic; @Builder(builderMethodName = "create") // not to conflict with super.builder() - public PartitionedQueueConsumerManager(QueueKey queueKey, String topic, long pollInterval, MsgPackProcessor msgPackProcessor, + public PartitionedQueueConsumerManager(Object queueKey, String topic, long pollInterval, MsgPackProcessor msgPackProcessor, BiFunction> consumerCreator, TbQueueAdmin queueAdmin, ExecutorService consumerExecutor, ScheduledExecutorService scheduler, ExecutorService taskExecutor, Consumer uncaughtErrorHandler) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/state/DefaultQueueStateService.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/state/DefaultQueueStateService.java index be019caaa7..be379fb76d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/state/DefaultQueueStateService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/state/DefaultQueueStateService.java @@ -18,10 +18,12 @@ package org.thingsboard.server.queue.common.state; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; +import java.util.Collections; + public class DefaultQueueStateService extends QueueStateService { public DefaultQueueStateService(PartitionedQueueConsumerManager eventConsumer) { - super(eventConsumer); + super(eventConsumer, Collections.emptyList()); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/state/KafkaQueueStateService.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/state/KafkaQueueStateService.java index bf02afe86c..d8d0c8e0d2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/state/KafkaQueueStateService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/state/KafkaQueueStateService.java @@ -22,6 +22,8 @@ import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; import org.thingsboard.server.queue.discovery.QueueKey; +import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Supplier; @@ -37,8 +39,9 @@ public class KafkaQueueStateService @Builder public KafkaQueueStateService(PartitionedQueueConsumerManager eventConsumer, PartitionedQueueConsumerManager stateConsumer, + List> otherConsumers, Supplier> eventsStartOffsetsProvider) { - super(eventConsumer); + super(eventConsumer, otherConsumers != null ? otherConsumers : Collections.emptyList()); this.stateConsumer = stateConsumer; this.eventsStartOffsetsProvider = eventsStartOffsetsProvider; } @@ -62,6 +65,9 @@ public class KafkaQueueStateService TopicPartitionInfo eventPartition = statePartition.withTopic(eventConsumer.getTopic()); if (this.partitions.get(queueKey).contains(eventPartition)) { eventConsumer.addPartitions(Set.of(eventPartition), null, eventsStartOffsets != null ? eventsStartOffsets::get : null); + for (PartitionedQueueConsumerManager consumer : otherConsumers) { + consumer.addPartitions(Set.of(statePartition.withTopic(consumer.getTopic()))); + } } } finally { readLock.unlock(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/state/QueueStateService.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/state/QueueStateService.java index 29426fab63..61b81707ce 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/state/QueueStateService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/state/QueueStateService.java @@ -25,6 +25,7 @@ import org.thingsboard.server.queue.discovery.QueueKey; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -37,6 +38,7 @@ import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTop public abstract class QueueStateService { protected final PartitionedQueueConsumerManager eventConsumer; + protected final List> otherConsumers; @Getter protected final Map> partitions = new HashMap<>(); @@ -45,8 +47,9 @@ public abstract class QueueStateService eventConsumer) { + protected QueueStateService(PartitionedQueueConsumerManager eventConsumer, List> otherConsumers) { this.eventConsumer = eventConsumer; + this.otherConsumers = otherConsumers; } public void update(QueueKey queueKey, Set newPartitions) { @@ -78,10 +81,16 @@ public abstract class QueueStateService partitions) { eventConsumer.addPartitions(partitions); + for (PartitionedQueueConsumerManager consumer : otherConsumers) { + consumer.addPartitions(withTopic(partitions, consumer.getTopic())); + } } protected void removePartitions(QueueKey queueKey, Set partitions) { eventConsumer.removePartitions(partitions); + for (PartitionedQueueConsumerManager consumer : otherConsumers) { + consumer.removePartitions(withTopic(partitions, consumer.getTopic())); + } } public void delete(Set partitions) { @@ -100,6 +109,9 @@ public abstract class QueueStateService partitions) { eventConsumer.delete(withTopic(partitions, eventConsumer.getTopic())); + for (PartitionedQueueConsumerManager consumer : otherConsumers) { + consumer.removePartitions(withTopic(partitions, consumer.getTopic())); + } } public Set getPartitionsInProgress() { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java index 3c927f135b..4a9b9f2a28 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java @@ -44,6 +44,8 @@ public class EdqsConfig { private int maxPendingRequests; @Value("${queue.edqs.max_request_timeout:20000}") private int maxRequestTimeout; + @Value("${queue.edqs.request_executor_size:50}") + private int requestExecutorSize; public String getLabel() { if (partitioningStrategy == EdqsPartitioningStrategy.NONE) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsExecutors.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsExecutors.java new file mode 100644 index 0000000000..8e804410fe --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsExecutors.java @@ -0,0 +1,70 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.common.util.ThingsBoardThreadFactory; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + +@Lazy +@Component +@Getter +@RequiredArgsConstructor +public class EdqsExecutors { + + private final EdqsConfig edqsConfig; + + private ExecutorService consumersExecutor; + private ExecutorService consumerTaskExecutor; + private ScheduledExecutorService scheduler; + private ListeningExecutorService requestExecutor; + + @PostConstruct + private void init() { + consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("edqs-consumer")); + consumerTaskExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "edqs-consumer-task-executor"); + scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-scheduler"); + requestExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(edqsConfig.getRequestExecutorSize(), "edqs-requests")); + } + + @PreDestroy + private void destroy() { + if (consumersExecutor != null) { + consumersExecutor.shutdownNow(); + } + if (consumerTaskExecutor != null) { + consumerTaskExecutor.shutdownNow(); + } + if (scheduler != null) { + scheduler.shutdownNow(); + } + if (requestExecutor != null) { + requestExecutor.shutdownNow(); + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java index 5c0d68779a..ed021e3380 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java @@ -19,8 +19,9 @@ import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueHandler; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.common.PartitionedQueueResponseTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; public interface EdqsQueueFactory { @@ -33,7 +34,7 @@ public interface EdqsQueueFactory { TbQueueProducer> createEdqsStateProducer(); - TbQueueResponseTemplate, TbProtoQueueMsg> createEdqsResponseTemplate(); + PartitionedQueueResponseTemplate, TbProtoQueueMsg> createEdqsResponseTemplate(TbQueueHandler, TbProtoQueueMsg> handler); TbQueueAdmin getEdqsQueueAdmin(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java index 8c670e66c0..6c7618f7db 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java @@ -17,16 +17,15 @@ package org.thingsboard.server.queue.edqs; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Component; -import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.stats.StatsType; import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueHandler; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueResponseTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate; +import org.thingsboard.server.queue.common.PartitionedQueueResponseTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.memory.InMemoryStorage; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; @@ -39,6 +38,7 @@ public class InMemoryEdqsQueueFactory implements EdqsQueueFactory { private final InMemoryStorage storage; private final EdqsConfig edqsConfig; + private final EdqsExecutors edqsExecutors; private final StatsFactory statsFactory; private final TbQueueAdmin queueAdmin; @@ -63,17 +63,21 @@ public class InMemoryEdqsQueueFactory implements EdqsQueueFactory { } @Override - public TbQueueResponseTemplate, TbProtoQueueMsg> createEdqsResponseTemplate() { - TbQueueConsumer> requestConsumer = new InMemoryTbQueueConsumer<>(storage, edqsConfig.getRequestsTopic()); + public PartitionedQueueResponseTemplate, TbProtoQueueMsg> createEdqsResponseTemplate(TbQueueHandler, TbProtoQueueMsg> handler) { TbQueueProducer> responseProducer = new InMemoryTbQueueProducer<>(storage, edqsConfig.getResponsesTopic()); - return DefaultTbQueueResponseTemplate., TbProtoQueueMsg>builder() - .requestTemplate(requestConsumer) - .responseTemplate(responseProducer) - .maxPendingRequests(edqsConfig.getMaxPendingRequests()) - .requestTimeout(edqsConfig.getMaxRequestTimeout()) + return PartitionedQueueResponseTemplate., TbProtoQueueMsg>builder() + .key("edqs") + .handler(handler) + .requestsTopic(edqsConfig.getRequestsTopic()) + .consumerCreator(tpi -> new InMemoryTbQueueConsumer<>(storage, edqsConfig.getRequestsTopic())) + .responseProducer(responseProducer) .pollInterval(edqsConfig.getPollInterval()) + .requestTimeout(edqsConfig.getMaxRequestTimeout()) + .maxPendingRequests(edqsConfig.getMaxPendingRequests()) + .consumerExecutor(edqsExecutors.getConsumersExecutor()) + .callbackExecutor(edqsExecutors.getRequestExecutor()) + .consumerTaskExecutor(edqsExecutors.getConsumerTaskExecutor()) .stats(statsFactory.createMessagesStats(StatsType.EDQS.getName())) - .executor(ThingsBoardExecutors.newWorkStealingPool(5, "edqs")) .build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java index ab88943b10..f071c30942 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java @@ -16,14 +16,13 @@ package org.thingsboard.server.queue.edqs; import org.springframework.stereotype.Component; -import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.stats.StatsType; import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueHandler; import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueResponseTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate; +import org.thingsboard.server.queue.common.PartitionedQueueResponseTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; @@ -45,6 +44,7 @@ public class KafkaEdqsQueueFactory implements EdqsQueueFactory { private final TbKafkaAdmin edqsRequestsAdmin; private final TbKafkaAdmin edqsStateAdmin; private final EdqsConfig edqsConfig; + private final EdqsExecutors edqsExecutors; private final TbServiceInfoProvider serviceInfoProvider; private final TbKafkaConsumerStatsService consumerStatsService; private final TopicService topicService; @@ -53,7 +53,7 @@ public class KafkaEdqsQueueFactory implements EdqsQueueFactory { private final AtomicInteger consumerCounter = new AtomicInteger(); public KafkaEdqsQueueFactory(TbKafkaSettings kafkaSettings, TbKafkaTopicConfigs topicConfigs, - EdqsConfig edqsConfig, TbServiceInfoProvider serviceInfoProvider, + EdqsConfig edqsConfig, EdqsExecutors edqsExecutors, TbServiceInfoProvider serviceInfoProvider, TbKafkaConsumerStatsService consumerStatsService, TopicService topicService, StatsFactory statsFactory) { this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, topicConfigs.getEdqsEventsConfigs()); @@ -61,6 +61,7 @@ public class KafkaEdqsQueueFactory implements EdqsQueueFactory { this.edqsStateAdmin = new TbKafkaAdmin(kafkaSettings, topicConfigs.getEdqsStateConfigs()); this.kafkaSettings = kafkaSettings; this.edqsConfig = edqsConfig; + this.edqsExecutors = edqsExecutors; this.serviceInfoProvider = serviceInfoProvider; this.consumerStatsService = consumerStatsService; this.topicService = topicService; @@ -116,25 +117,29 @@ public class KafkaEdqsQueueFactory implements EdqsQueueFactory { } @Override - public TbQueueResponseTemplate, TbProtoQueueMsg> createEdqsResponseTemplate() { - var requestConsumer = createEdqsMsgConsumer(edqsConfig.getRequestsTopic(), - "edqs-requests-consumer-" + serviceInfoProvider.getServiceId(), - "edqs-requests-consumer-group", - false, edqsRequestsAdmin); + public PartitionedQueueResponseTemplate, TbProtoQueueMsg> createEdqsResponseTemplate(TbQueueHandler, TbProtoQueueMsg> handler) { var responseProducer = TbKafkaProducerTemplate.>builder() .settings(kafkaSettings) .clientId("edqs-response-producer-" + serviceInfoProvider.getServiceId()) .defaultTopic(topicService.buildTopicName(edqsConfig.getResponsesTopic())) .admin(edqsRequestsAdmin) .build(); - return DefaultTbQueueResponseTemplate., TbProtoQueueMsg>builder() - .requestTemplate(requestConsumer) - .responseTemplate(responseProducer) - .maxPendingRequests(edqsConfig.getMaxPendingRequests()) - .requestTimeout(edqsConfig.getMaxRequestTimeout()) + return PartitionedQueueResponseTemplate., TbProtoQueueMsg>builder() + .key("edqs") + .handler(handler) + .requestsTopic(topicService.buildTopicName(edqsConfig.getRequestsTopic())) + .consumerCreator(tpi -> createEdqsMsgConsumer(edqsConfig.getRequestsTopic(), + "edqs-requests-consumer-" + serviceInfoProvider.getServiceId() + "-" + tpi.getPartition().orElse(999), + "edqs-requests-consumer-group", + false, edqsRequestsAdmin)) + .responseProducer(responseProducer) .pollInterval(edqsConfig.getPollInterval()) + .requestTimeout(edqsConfig.getMaxRequestTimeout()) + .maxPendingRequests(edqsConfig.getMaxPendingRequests()) + .consumerExecutor(edqsExecutors.getConsumersExecutor()) + .callbackExecutor(edqsExecutors.getRequestExecutor()) + .consumerTaskExecutor(edqsExecutors.getConsumerTaskExecutor()) .stats(statsFactory.createMessagesStats(StatsType.EDQS.getName())) - .executor(ThingsBoardExecutors.newWorkStealingPool(5, "edqs")) .build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java index cac6f2ea1e..f0b3694d74 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java @@ -75,10 +75,6 @@ public class TbKafkaProducerTemplate implements TbQueuePro topics = ConcurrentHashMap.newKeySet(); } - @Override - public void init() { - } - void addAnalyticHeaders(List
headers) { headers.add(new RecordHeader("_producerId", getClientId().getBytes(StandardCharsets.UTF_8))); headers.add(new RecordHeader("_threadName", Thread.currentThread().getName().getBytes(StandardCharsets.UTF_8))); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java index 49c52ed21a..43d820bc7e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java @@ -33,11 +33,6 @@ public class InMemoryTbQueueProducer implements TbQueuePro this.defaultTopic = defaultTopic; } - @Override - public void init() { - - } - @Override public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { boolean result = storage.put(tpi.getFullTopicName(), msg); diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index 9e493220ca..e5bb62bc04 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -37,11 +37,23 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThan; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.BDDMockito.*; -import static org.mockito.Mockito.*; +import static org.mockito.BDDMockito.RETURNS_DEEP_STUBS; +import static org.mockito.BDDMockito.atLeastOnce; +import static org.mockito.BDDMockito.lenient; +import static org.mockito.BDDMockito.mock; +import static org.mockito.BDDMockito.never; +import static org.mockito.BDDMockito.spy; +import static org.mockito.BDDMockito.times; +import static org.mockito.BDDMockito.verify; +import static org.mockito.BDDMockito.willAnswer; +import static org.mockito.BDDMockito.willDoNothing; +import static org.mockito.BDDMockito.willReturn; import static org.mockito.hamcrest.MockitoHamcrest.longThat; @Slf4j @@ -96,7 +108,6 @@ public class DefaultTbQueueRequestTemplateTest { inst.init(); assertThat(inst.nextCleanupNs, equalTo(0L)); verify(queueAdmin, times(1)).createTopicIfNotExists(topic); - verify(requestTemplate, times(1)).init(); verify(responseTemplate, times(1)).subscribe(); verify(executorMock, times(1)).submit(any(Runnable.class)); diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml index 1cc32a4230..0815187704 100644 --- a/edqs/src/main/resources/edqs.yml +++ b/edqs/src/main/resources/edqs.yml @@ -71,6 +71,8 @@ queue: max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}" # Maximum timeout for requests to EDQS max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:20000}" + # Thread pool size for EDQS requests executor + request_executor_size: "${TB_EDQS_REQUEST_EXECUTOR_SIZE:50}" # Strings longer than this threshold will be compressed string_compression_length_threshold: "${TB_EDQS_STRING_COMPRESSION_LENGTH_THRESHOLD:512}" stats: From 26d949d22fc8443ca06bfdff966da80608ddae0b Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 16 Apr 2025 16:18:21 +0200 Subject: [PATCH 05/79] invalidate async js scripts --- .../api/AbstractScriptInvokeService.java | 4 +- .../api/js/AbstractJsInvokeService.java | 17 +++ .../script/api/js/JsValidator.java | 120 ++++++++++++++++++ .../api/js/AbstractJsInvokeServiceTest.java | 91 +++++++++++++ .../script/api/js/JsValidatorTest.java | 88 +++++++++++++ 5 files changed, 318 insertions(+), 2 deletions(-) create mode 100644 common/script/script-api/src/main/java/org/thingsboard/script/api/js/JsValidator.java create mode 100644 common/script/script-api/src/test/java/org/thingsboard/script/api/js/AbstractJsInvokeServiceTest.java create mode 100644 common/script/script-api/src/test/java/org/thingsboard/script/api/js/JsValidatorTest.java diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/AbstractScriptInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/AbstractScriptInvokeService.java index 3d9b855064..f7b640d3af 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/AbstractScriptInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/AbstractScriptInvokeService.java @@ -269,7 +269,7 @@ public abstract class AbstractScriptInvokeService implements ScriptInvokeService } } - private boolean scriptBodySizeExceeded(String scriptBody) { + public boolean scriptBodySizeExceeded(String scriptBody) { if (getMaxScriptBodySize() <= 0) return false; return scriptBody.length() > getMaxScriptBodySize(); } @@ -297,7 +297,7 @@ public abstract class AbstractScriptInvokeService implements ScriptInvokeService return result != null && result.length() > getMaxResultSize(); } - private ListenableFuture error(String message) { + public ListenableFuture error(String message) { return Futures.immediateFailedFuture(new RuntimeException(message)); } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/js/AbstractJsInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/js/AbstractJsInvokeService.java index b6ddf16e66..d8fae31290 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/js/AbstractJsInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/js/AbstractJsInvokeService.java @@ -35,6 +35,8 @@ import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import static java.lang.String.format; + /** * Created by ashvayka on 26.09.18. */ @@ -93,6 +95,21 @@ public abstract class AbstractJsInvokeService extends AbstractScriptInvokeServic doRelease(scriptId, scriptInfoMap.remove(scriptId)); } + @Override + public ListenableFuture eval(TenantId tenantId, ScriptType scriptType, String scriptBody, String... argNames) { + if (!isExecEnabled(tenantId)) { + return error("Script Execution is disabled due to API limits!"); + } + if (scriptBodySizeExceeded(scriptBody)) { + return error(format("Script body exceeds maximum allowed size of %s symbols", getMaxScriptBodySize())); + } + final String validationIssue = JsValidator.validate(scriptBody); + if (validationIssue != null ) { + return error(validationIssue); + } + return super.eval(tenantId, scriptType, scriptBody, argNames); + } + protected abstract ListenableFuture doEval(UUID scriptId, JsScriptInfo jsInfo, String scriptBody); protected abstract ListenableFuture doInvokeFunction(UUID scriptId, JsScriptInfo jsInfo, Object[] args); diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/js/JsValidator.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/js/JsValidator.java new file mode 100644 index 0000000000..9c65064802 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/js/JsValidator.java @@ -0,0 +1,120 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.js; + +import java.util.regex.Pattern; + +public class JsValidator { + + static final Pattern ASYNC_PATTERN = Pattern.compile("\\basync\\b"); + static final Pattern AWAIT_PATTERN = Pattern.compile("\\bawait\\b"); + static final Pattern PROMISE_PATTERN = Pattern.compile("\\bPromise\\b"); + static final Pattern SET_TIMEOUT_PATTERN = Pattern.compile("\\bsetTimeout\\b"); + + public static String validate(String scriptBody) { + if (scriptBody == null || scriptBody.trim().isEmpty()) { + return "Script body is empty"; + } + + //Quick check + if (!ASYNC_PATTERN.matcher(scriptBody).find() + && !AWAIT_PATTERN.matcher(scriptBody).find() + && !PROMISE_PATTERN.matcher(scriptBody).find() + && !SET_TIMEOUT_PATTERN.matcher(scriptBody).find()) { + return null; + } + + //Recheck if quick check failed. Ignoring comments and strings + String[] lines = scriptBody.split("\\r?\\n"); + boolean insideMultilineComment = false; + + for (String line : lines) { + String stripped = line; + + // Handle multiline comments + if (insideMultilineComment) { + if (line.contains("*/")) { + insideMultilineComment = false; + stripped = line.substring(line.indexOf("*/") + 2); // continue after comment + } else { + continue; // skip line inside multiline comment + } + } + + // Check for start of multiline comment + if (stripped.contains("/*")) { + int start = stripped.indexOf("/*"); + int end = stripped.indexOf("*/", start + 2); + + if (end != -1) { + // Inline multiline comment + stripped = stripped.substring(0, start) + stripped.substring(end + 2); + } else { + // Starts a block comment, continues on next lines + insideMultilineComment = true; + stripped = stripped.substring(0, start); + } + } + + stripped = stripInlineComment(stripped); + stripped = stripStringLiterals(stripped); + + if (ASYNC_PATTERN.matcher(stripped).find()) { + return "Script must not contain 'async' keyword."; + } + if (AWAIT_PATTERN.matcher(stripped).find()) { + return "Script must not contain 'await' keyword."; + } + if (PROMISE_PATTERN.matcher(stripped).find()) { + return "Script must not use 'Promise'."; + } + if (SET_TIMEOUT_PATTERN.matcher(stripped).find()) { + return "Script must not use 'setTimeout' method."; + } + } + return null; + } + + private static String stripInlineComment(String line) { + int index = line.indexOf("//"); + return index >= 0 ? line.substring(0, index) : line; + } + + private static String stripStringLiterals(String line) { + StringBuilder sb = new StringBuilder(); + boolean inSingleQuote = false; + boolean inDoubleQuote = false; + + for (int i = 0; i < line.length(); i++) { + char c = line.charAt(i); + + if (c == '"' && !inSingleQuote) { + inDoubleQuote = !inDoubleQuote; + continue; + } else if (c == '\'' && !inDoubleQuote) { + inSingleQuote = !inSingleQuote; + continue; + } + + if (!inSingleQuote && !inDoubleQuote) { + sb.append(c); + } + } + + return sb.toString(); + } + +} diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/js/AbstractJsInvokeServiceTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/js/AbstractJsInvokeServiceTest.java new file mode 100644 index 0000000000..760ca9e3fb --- /dev/null +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/js/AbstractJsInvokeServiceTest.java @@ -0,0 +1,91 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.js; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import org.thingsboard.server.common.stats.StatsCounter; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.script.api.ScriptType; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doCallRealMethod; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +@Slf4j +class AbstractJsInvokeServiceTest { + + AbstractJsInvokeService service; + final UUID id = UUID.randomUUID(); + + @BeforeEach + void setUp() { + service = mock(AbstractJsInvokeService.class, Mockito.RETURNS_DEEP_STUBS); + + ReflectionTestUtils.setField(service, "requestsCounter", mock(StatsCounter.class)); + ReflectionTestUtils.setField(service, "evalCallback", mock(FutureCallback.class)); + + // Make sure core checks always pass + doReturn(true).when(service).isExecEnabled(any()); + doReturn(false).when(service).scriptBodySizeExceeded(anyString()); + doReturn(Futures.immediateFuture(id)).when(service).doEvalScript(any(), any(), anyString(), any(), any(String[].class)); + + // Use real implementation of eval() + doCallRealMethod().when(service).eval(any(), any(), any(), any(String[].class)); + doCallRealMethod().when(service).error(anyString()); + } + + @Test + void shouldReturnValidationErrorFromJsValidator() throws ExecutionException, InterruptedException { + String scriptWithAsync = "async function test() {}"; + + var future = service.eval(TenantId.SYS_TENANT_ID, ScriptType.RULE_NODE_SCRIPT, scriptWithAsync, "a", "b"); + ExecutionException ex = assertThrows(ExecutionException.class, future::get); + assertTrue(ex.getCause().getMessage().contains("Script must not contain 'async' keyword.")); + assertThat(ex.getCause()).isInstanceOf(RuntimeException.class); + verify(service).isExecEnabled(any()); + verify(service).scriptBodySizeExceeded(any()); + } + + @Test + void shouldPassValidationAndCallSuperEval() throws ExecutionException, InterruptedException, TimeoutException { + String validScript = "function test() { return 42; }"; + var result = service.eval(TenantId.SYS_TENANT_ID, ScriptType.RULE_NODE_SCRIPT, validScript, "x", "y"); + + assertThat(result.get(30, TimeUnit.SECONDS)).isEqualTo(id); + // two times, non-optimal + verify(service, times(2)).isExecEnabled(any()); + verify(service, times(2)).scriptBodySizeExceeded(any()); + } + +} diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/js/JsValidatorTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/js/JsValidatorTest.java new file mode 100644 index 0000000000..c1dbf7a58b --- /dev/null +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/js/JsValidatorTest.java @@ -0,0 +1,88 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.js; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +class JsValidatorTest { + + @ParameterizedTest(name = "should return error for script \"{0}\"") + @ValueSource(strings = { + "async function test() {}", + "const result = await someFunc();", + "const result =\nawait\tsomeFunc();", + "setTimeout(1000);", + "new Promise((resolve) => {});", + "function test() { return 42; } \n\t await test()", + """ + function init() { + await doSomething(); + } + """, + }) + void shouldReturnErrorForInvalidScripts(String script) { + assertNotNull(JsValidator.validate(script)); + } + + @ParameterizedTest(name = "should pass validation for script: \"{0}\"") + @ValueSource(strings = { + "function test() { return 42; }", + "const result = 10 * 2;", + "// async is a keyword but not used: 'const word = \"async\";'", + "let note = 'setTimeout tight';", + + "const word = \"async\";", + "const word = \"setTimeout\";", + "const word = \"Promise\";", + "const word = \"await\";", + + "const word = 'async';", + "const word = 'setTimeout';", + "const word = 'Promise';", + "const word = 'await';", + + "//function test() { return 42; }", + "// const result = 10 * 2;", + "// async is a keyword but not used: 'const word = \"async\";'", + "//setTimeout(1);", + + "a=b+c; // await for a day", + "return new // Promise((resolve) => {", + "hello(); // async is a keyword but not used: 'const word = \"async\";'", + "setGoal(a); //setTimeout(1);", + + " /* new Promise((resolve) => {}); // */ return 'await';", + " /* async */ function calc() {", + "/* async function abc() { \n await new Promise ( \t setTimeout () ) \n } \n*/", + }) + void shouldReturnNullForValidScripts(String script) { + assertNull(JsValidator.validate(script)); + } + + @ParameterizedTest(name = "should return 'Script body is empty' for input: \"{0}\"") + @NullAndEmptySource + @ValueSource(strings = {" ", "\t", "\n"}) + void shouldReturnErrorForEmptyOrNullScripts(String script) { + assertEquals("Script body is empty", JsValidator.validate(script)); + } + +} From 5a46a170e45c67f77ba7062579400ba421618c78 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 16 Apr 2025 16:47:20 +0200 Subject: [PATCH 06/79] MQTT client log added: Scheduling reconnect in [{}] sec --- .../src/main/java/org/thingsboard/mqtt/MqttClientImpl.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java index 344886165e..59a680662f 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java @@ -194,7 +194,10 @@ final class MqttClientImpl implements MqttClient { if (reconnect) { this.reconnect = true; } - eventLoop.schedule((Runnable) () -> connect(host, port, reconnect), reconnectStrategy.getNextReconnectDelay(), TimeUnit.SECONDS); + + final long nextReconnectDelay = reconnectStrategy.getNextReconnectDelay(); + log.info("[{}] Scheduling reconnect in [{}] sec", channel != null ? channel.id() : "UNKNOWN", nextReconnectDelay); + eventLoop.schedule((Runnable) () -> connect(host, port, reconnect), nextReconnectDelay, TimeUnit.SECONDS); } } From 2a50e2eaa53b445488154297ab67bdcc9a672b30 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 22 Apr 2025 16:38:52 +0200 Subject: [PATCH 07/79] AbstractScriptInvokeService: validate script refactored --- .../api/AbstractScriptInvokeService.java | 26 ++-- .../api/js/AbstractJsInvokeService.java | 16 +-- .../api/AbstractScriptInvokeServiceTest.java | 122 ++++++++++++++++++ .../api/js/AbstractJsInvokeServiceTest.java | 8 +- 4 files changed, 149 insertions(+), 23 deletions(-) create mode 100644 common/script/script-api/src/test/java/org/thingsboard/script/api/AbstractScriptInvokeServiceTest.java diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/AbstractScriptInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/AbstractScriptInvokeService.java index f7b640d3af..0b3890b4bc 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/AbstractScriptInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/AbstractScriptInvokeService.java @@ -134,19 +134,29 @@ public abstract class AbstractScriptInvokeService implements ScriptInvokeService } } - @Override - public ListenableFuture eval(TenantId tenantId, ScriptType scriptType, String scriptBody, String... argNames) { + public String validate(TenantId tenantId, String scriptBody) { if (isExecEnabled(tenantId)) { if (scriptBodySizeExceeded(scriptBody)) { - return error(format("Script body exceeds maximum allowed size of %s symbols", getMaxScriptBodySize())); + return format("Script body exceeds maximum allowed size of %s symbols", getMaxScriptBodySize()); } - UUID scriptId = UUID.randomUUID(); - requestsCounter.increment(); - return withTimeoutAndStatsCallback(scriptId, null, - doEvalScript(tenantId, scriptType, scriptBody, scriptId, argNames), evalCallback, getMaxEvalRequestsTimeout()); } else { - return error("Script Execution is disabled due to API limits!"); + return "Script Execution is disabled due to API limits!"; } + + return null; + } + + @Override + public ListenableFuture eval(TenantId tenantId, ScriptType scriptType, String scriptBody, String... argNames) { + String validationError = validate(tenantId, scriptBody); + if (validationError != null) { + return error(validationError); + } + + UUID scriptId = UUID.randomUUID(); + requestsCounter.increment(); + return withTimeoutAndStatsCallback(scriptId, null, + doEvalScript(tenantId, scriptType, scriptBody, scriptId, argNames), evalCallback, getMaxEvalRequestsTimeout()); } @Override diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/js/AbstractJsInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/js/AbstractJsInvokeService.java index d8fae31290..c859f49629 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/js/AbstractJsInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/js/AbstractJsInvokeService.java @@ -96,18 +96,12 @@ public abstract class AbstractJsInvokeService extends AbstractScriptInvokeServic } @Override - public ListenableFuture eval(TenantId tenantId, ScriptType scriptType, String scriptBody, String... argNames) { - if (!isExecEnabled(tenantId)) { - return error("Script Execution is disabled due to API limits!"); + public String validate(TenantId tenantId, String scriptBody) { + String errorMessage = super.validate(tenantId, scriptBody); + if (errorMessage == null) { + return JsValidator.validate(scriptBody); } - if (scriptBodySizeExceeded(scriptBody)) { - return error(format("Script body exceeds maximum allowed size of %s symbols", getMaxScriptBodySize())); - } - final String validationIssue = JsValidator.validate(scriptBody); - if (validationIssue != null ) { - return error(validationIssue); - } - return super.eval(tenantId, scriptType, scriptBody, argNames); + return errorMessage; } protected abstract ListenableFuture doEval(UUID scriptId, JsScriptInfo jsInfo, String scriptBody); diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/AbstractScriptInvokeServiceTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/AbstractScriptInvokeServiceTest.java new file mode 100644 index 0000000000..35f201ed3b --- /dev/null +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/AbstractScriptInvokeServiceTest.java @@ -0,0 +1,122 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.stats.StatsCounter; + +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doCallRealMethod; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +class AbstractScriptInvokeServiceTest { + + AbstractScriptInvokeService service; + final UUID id = UUID.randomUUID(); + final String scriptBody = "return true;"; + final TenantId tenantId = TenantId.fromUUID(UUID.fromString("2ed9a658-45a5-4812-b212-9931f5749f30")); + + @BeforeEach + void setUp() { + service = mock(AbstractScriptInvokeService.class, Mockito.RETURNS_DEEP_STUBS); + + // Make sure core checks always pass + doReturn(true).when(service).isExecEnabled(any()); + doReturn(50000L).when(service).getMaxScriptBodySize(); + + // Use real implementations + doCallRealMethod().when(service).scriptBodySizeExceeded(anyString()); + doCallRealMethod().when(service).eval(any(), any(), any(), any(String[].class)); + doCallRealMethod().when(service).error(anyString()); + doCallRealMethod().when(service).validate(any(), anyString()); + } + + @Test + void evalWithValidationCallTest() throws ExecutionException, InterruptedException, TimeoutException { + ReflectionTestUtils.setField(service, "requestsCounter", mock(StatsCounter.class)); + ReflectionTestUtils.setField(service, "evalCallback", mock(FutureCallback.class)); + + doReturn(Futures.immediateFuture(id)).when(service).doEvalScript(any(), any(), anyString(), any(), any(String[].class)); + + var future = service.eval(tenantId, ScriptType.RULE_NODE_SCRIPT, scriptBody, "x", "y"); + + assertThat(future.get(30, TimeUnit.SECONDS)).isEqualTo(id); + verify(service).validate(any(), anyString()); + verify(service).validate(tenantId, scriptBody); + verify(service, never()).error(anyString()); + } + + @Test + void evalWithValidationCallErrorTest() throws ExecutionException, InterruptedException, TimeoutException { + doReturn(false).when(service).isExecEnabled(any()); + var future = service.eval(tenantId, ScriptType.RULE_NODE_SCRIPT, scriptBody, "x", "y"); + + ExecutionException ex = assertThrows(ExecutionException.class, future::get); + assertThat(ex.getCause().getMessage()).isEqualTo("Script Execution is disabled due to API limits!"); + assertThat(ex.getCause()).isInstanceOf(RuntimeException.class); + + verify(service).validate(any(), anyString()); + verify(service).validate(tenantId, scriptBody); + verify(service).error(anyString()); + } + + @Test + void validateScriptBodyTestExecEnabledTest() { + assertNull(service.validate(tenantId, scriptBody)); + verify(service).isExecEnabled(tenantId); + } + + @Test + void validateScriptBodyTestExecDisabledTest() { + doReturn(false).when(service).isExecEnabled(tenantId); + assertThat(service.validate(tenantId, scriptBody)).isEqualTo("Script Execution is disabled due to API limits!"); + verify(service).isExecEnabled(tenantId); + } + + @Test + void validateScriptBodySizeOKTest() { + assertNull(service.validate(tenantId, scriptBody)); + verify(service).isExecEnabled(tenantId); + verify(service).scriptBodySizeExceeded(scriptBody); + } + + @Test + void validateScriptBodySizeExceededTest() { + doReturn(10L).when(service).getMaxScriptBodySize(); + assertThat(service.validate(tenantId, scriptBody)).isEqualTo("Script body exceeds maximum allowed size of 10 symbols"); + verify(service).isExecEnabled(tenantId); + verify(service).scriptBodySizeExceeded(scriptBody); + } + +} diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/js/AbstractJsInvokeServiceTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/js/AbstractJsInvokeServiceTest.java index 760ca9e3fb..3f7d61919d 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/js/AbstractJsInvokeServiceTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/js/AbstractJsInvokeServiceTest.java @@ -60,9 +60,10 @@ class AbstractJsInvokeServiceTest { doReturn(false).when(service).scriptBodySizeExceeded(anyString()); doReturn(Futures.immediateFuture(id)).when(service).doEvalScript(any(), any(), anyString(), any(), any(String[].class)); - // Use real implementation of eval() + // Use real implementations doCallRealMethod().when(service).eval(any(), any(), any(), any(String[].class)); doCallRealMethod().when(service).error(anyString()); + doCallRealMethod().when(service).validate(any(), anyString()); } @Test @@ -83,9 +84,8 @@ class AbstractJsInvokeServiceTest { var result = service.eval(TenantId.SYS_TENANT_ID, ScriptType.RULE_NODE_SCRIPT, validScript, "x", "y"); assertThat(result.get(30, TimeUnit.SECONDS)).isEqualTo(id); - // two times, non-optimal - verify(service, times(2)).isExecEnabled(any()); - verify(service, times(2)).scriptBodySizeExceeded(any()); + verify(service, times(1)).isExecEnabled(any()); + verify(service, times(1)).scriptBodySizeExceeded(any()); } } From 49289da5a8e15e99d4a8776f935a2f38a2d6f1fc Mon Sep 17 00:00:00 2001 From: Tarnavskiy Date: Wed, 23 Apr 2025 17:55:51 +0300 Subject: [PATCH 08/79] Fixed an issue when the user could break the pagination settings validation by switching between the basic/advanced mode tabs in table-widgets --- .../widget/lib/alarm/alarms-table-widget.component.ts | 8 +++++--- .../widget/lib/entity/entities-table-widget.component.ts | 8 +++++--- .../widget/lib/rpc/persistent-table.component.ts | 8 +++++--- .../home/components/widget/lib/table-widget.models.ts | 6 +++--- .../widget/lib/timeseries-table-widget.component.ts | 8 +++++--- 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts index 7384f76b62..31843506cc 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts @@ -392,10 +392,12 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, this.rowStylesInfo = getRowStyleInfo(this.ctx, this.settings, 'alarm, ctx'); const pageSize = this.settings.defaultPageSize; - let pageStepIncrement = this.settings.pageStepIncrement; - let pageStepCount = this.settings.pageStepCount; + let pageStepIncrement = Number.isInteger(this.settings.pageStepIncrement) && this.settings.pageStepIncrement > 0 ? + this.settings.pageStepIncrement : null; + let pageStepCount = Number.isInteger(this.settings.pageStepCount) && this.settings.pageStepCount > 0 + && this.settings.pageStepCount <= 100 ? this.settings.pageStepCount : null; - if (isDefined(pageSize) && isNumber(pageSize) && pageSize > 0) { + if (Number.isInteger(pageSize) && pageSize > 0) { this.defaultPageSize = pageSize; } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/entity/entities-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/entity/entities-table-widget.component.ts index 653c388364..a80c6896bb 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/entity/entities-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/entity/entities-table-widget.component.ts @@ -311,10 +311,12 @@ export class EntitiesTableWidgetComponent extends PageComponent implements OnIni this.rowStylesInfo = getRowStyleInfo(this.ctx, this.settings, 'entity, ctx'); const pageSize = this.settings.defaultPageSize; - let pageStepIncrement = this.settings.pageStepIncrement; - let pageStepCount = this.settings.pageStepCount; + let pageStepIncrement = Number.isInteger(this.settings.pageStepIncrement) && this.settings.pageStepIncrement > 0 ? + this.settings.pageStepIncrement : null; + let pageStepCount = Number.isInteger(this.settings.pageStepCount) && this.settings.pageStepCount > 0 + && this.settings.pageStepCount <= 100 ? this.settings.pageStepCount : null; - if (isDefined(pageSize) && isNumber(pageSize) && pageSize > 0) { + if (Number.isInteger(pageSize) && pageSize > 0) { this.defaultPageSize = pageSize; } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/persistent-table.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/persistent-table.component.ts index 586f425b13..0647f41771 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/persistent-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/persistent-table.component.ts @@ -207,10 +207,12 @@ export class PersistentTableComponent extends PageComponent implements OnInit, O this.displayedColumns = [...this.displayTableColumns]; const pageSize = this.settings.defaultPageSize; - let pageStepIncrement = this.settings.pageStepIncrement; - let pageStepCount = this.settings.pageStepCount; + let pageStepIncrement = Number.isInteger(this.settings.pageStepIncrement) && this.settings.pageStepIncrement > 0 ? + this.settings.pageStepIncrement : null; + let pageStepCount = Number.isInteger(this.settings.pageStepCount) && this.settings.pageStepCount > 0 + && this.settings.pageStepCount <= 100 ? this.settings.pageStepCount : null; - if (isDefined(pageSize) && isNumber(pageSize) && pageSize > 0) { + if (Number.isInteger(pageSize) && pageSize > 0) { this.defaultPageSize = pageSize; } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.models.ts index 8aadc62ee4..eb944e3c65 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.models.ts @@ -16,7 +16,7 @@ import { EntityId } from '@shared/models/id/entity-id'; import { DataKey, FormattedData, WidgetActionDescriptor, WidgetConfig } from '@shared/models/widget.models'; -import { getDescendantProp, isDefined, isDefinedAndNotNull, isNotEmptyStr } from '@core/utils'; +import { getDescendantProp, isDefined, isNotEmptyStr } from '@core/utils'; import { AlarmDataInfo, alarmFields } from '@shared/models/alarm.models'; import tinycolor from 'tinycolor2'; import { Direction } from '@shared/models/page/sort-order'; @@ -564,8 +564,8 @@ export function getHeaderTitle(dataKey: DataKey, keySettings: TableWidgetDataKey export function buildPageStepSizeValues(pageStepCount: number, pageStepIncrement: number): Array { const pageSteps: Array = []; - if (isDefinedAndNotNull(pageStepCount) && pageStepCount > 0 && pageStepCount <= 100 && - isDefinedAndNotNull(pageStepIncrement) && pageStepIncrement > 0) { + if (Number.isInteger(pageStepCount) && pageStepCount > 0 && pageStepCount <= 100 && + Number.isInteger(pageStepIncrement) && pageStepIncrement > 0) { for (let i = 1; i <= pageStepCount; i++) { pageSteps.push(pageStepIncrement * i); } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts index 2653b79059..010e49fe32 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts @@ -352,10 +352,12 @@ export class TimeseriesTableWidgetComponent extends PageComponent implements OnI this.rowStylesInfo = getRowStyleInfo(this.ctx, this.settings, 'rowData, ctx'); const pageSize = this.settings.defaultPageSize; - let pageStepIncrement = this.settings.pageStepIncrement; - let pageStepCount = this.settings.pageStepCount; + let pageStepIncrement = Number.isInteger(this.settings.pageStepIncrement) && this.settings.pageStepIncrement > 0 ? + this.settings.pageStepIncrement : null; + let pageStepCount = Number.isInteger(this.settings.pageStepCount) && this.settings.pageStepCount > 0 + && this.settings.pageStepCount <= 100 ? this.settings.pageStepCount : null; - if (isDefined(pageSize) && isNumber(pageSize) && pageSize > 0) { + if (Number.isInteger(pageSize) && pageSize > 0) { this.defaultPageSize = pageSize; } From 358e635805dfde918eedc4543f839d8e3115a9ce Mon Sep 17 00:00:00 2001 From: Tarnavskiy Date: Wed, 23 Apr 2025 18:34:58 +0300 Subject: [PATCH 09/79] Code optimization for a fix of an issue with broken pagination settings validation in table-widgets --- .../widget/lib/alarm/alarms-table-widget.component.ts | 9 ++++----- .../lib/entity/entities-table-widget.component.ts | 10 +++++----- .../widget/lib/rpc/persistent-table.component.ts | 10 +++++----- .../home/components/widget/lib/table-widget.models.ts | 11 +++++++++-- .../widget/lib/timeseries-table-widget.component.ts | 9 ++++----- 5 files changed, 27 insertions(+), 22 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts index 31843506cc..5321e27c05 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/alarm/alarms-table-widget.component.ts @@ -43,7 +43,6 @@ import { isDefined, isDefinedAndNotNull, isNotEmptyStr, - isNumber, isObject, isUndefined } from '@core/utils'; @@ -77,6 +76,8 @@ import { getHeaderTitle, getRowStyleInfo, getTableCellButtonActions, + isValidPageStepCount, + isValidPageStepIncrement, noDataMessage, prepareTableCellButtonActions, RowStyleInfo, @@ -392,10 +393,8 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, this.rowStylesInfo = getRowStyleInfo(this.ctx, this.settings, 'alarm, ctx'); const pageSize = this.settings.defaultPageSize; - let pageStepIncrement = Number.isInteger(this.settings.pageStepIncrement) && this.settings.pageStepIncrement > 0 ? - this.settings.pageStepIncrement : null; - let pageStepCount = Number.isInteger(this.settings.pageStepCount) && this.settings.pageStepCount > 0 - && this.settings.pageStepCount <= 100 ? this.settings.pageStepCount : null; + let pageStepIncrement = isValidPageStepIncrement(this.settings.pageStepIncrement) ? this.settings.pageStepIncrement : null; + let pageStepCount = isValidPageStepCount(this.settings.pageStepCount) ? this.settings.pageStepCount : null; if (Number.isInteger(pageSize) && pageSize > 0) { this.defaultPageSize = pageSize; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/entity/entities-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/entity/entities-table-widget.component.ts index a80c6896bb..2a87af2519 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/entity/entities-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/entity/entities-table-widget.component.ts @@ -42,7 +42,7 @@ import { import { IWidgetSubscription } from '@core/api/widget-api.models'; import { UtilsService } from '@core/services/utils.service'; import { TranslateService } from '@ngx-translate/core'; -import { deepClone, hashCode, isDefined, isDefinedAndNotNull, isNumber, isObject, isUndefined } from '@core/utils'; +import { deepClone, hashCode, isDefined, isDefinedAndNotNull, isObject, isUndefined } from '@core/utils'; import cssjs from '@core/css/css'; import { CollectionViewer, DataSource } from '@angular/cdk/collections'; import { DataKeyType } from '@shared/models/telemetry/telemetry.models'; @@ -75,6 +75,8 @@ import { getHeaderTitle, getRowStyleInfo, getTableCellButtonActions, + isValidPageStepCount, + isValidPageStepIncrement, noDataMessage, prepareTableCellButtonActions, RowStyleInfo, @@ -311,10 +313,8 @@ export class EntitiesTableWidgetComponent extends PageComponent implements OnIni this.rowStylesInfo = getRowStyleInfo(this.ctx, this.settings, 'entity, ctx'); const pageSize = this.settings.defaultPageSize; - let pageStepIncrement = Number.isInteger(this.settings.pageStepIncrement) && this.settings.pageStepIncrement > 0 ? - this.settings.pageStepIncrement : null; - let pageStepCount = Number.isInteger(this.settings.pageStepCount) && this.settings.pageStepCount > 0 - && this.settings.pageStepCount <= 100 ? this.settings.pageStepCount : null; + let pageStepIncrement = isValidPageStepIncrement(this.settings.pageStepIncrement) ? this.settings.pageStepIncrement : null; + let pageStepCount = isValidPageStepCount(this.settings.pageStepCount) ? this.settings.pageStepCount : null; if (Number.isInteger(pageSize) && pageSize > 0) { this.defaultPageSize = pageSize; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/persistent-table.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/persistent-table.component.ts index 0647f41771..60f522b192 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/persistent-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/persistent-table.component.ts @@ -36,6 +36,8 @@ import { BehaviorSubject, merge, Observable, of, ReplaySubject, Subject, throwEr import { catchError, map, tap } from 'rxjs/operators'; import { constructTableCssString, + isValidPageStepCount, + isValidPageStepIncrement, noDataMessage, TableCellButtonActionDescriptor, TableWidgetSettings @@ -43,7 +45,7 @@ import { import cssjs from '@core/css/css'; import { UtilsService } from '@core/services/utils.service'; import { TranslateService } from '@ngx-translate/core'; -import { hashCode, isDefined, isDefinedAndNotNull, isNumber, parseHttpErrorMessage } from '@core/utils'; +import { hashCode, isDefined, isDefinedAndNotNull, parseHttpErrorMessage } from '@core/utils'; import { CollectionViewer, DataSource } from '@angular/cdk/collections'; import { emptyPageData, PageData } from '@shared/models/page/page-data'; import { @@ -207,10 +209,8 @@ export class PersistentTableComponent extends PageComponent implements OnInit, O this.displayedColumns = [...this.displayTableColumns]; const pageSize = this.settings.defaultPageSize; - let pageStepIncrement = Number.isInteger(this.settings.pageStepIncrement) && this.settings.pageStepIncrement > 0 ? - this.settings.pageStepIncrement : null; - let pageStepCount = Number.isInteger(this.settings.pageStepCount) && this.settings.pageStepCount > 0 - && this.settings.pageStepCount <= 100 ? this.settings.pageStepCount : null; + let pageStepIncrement = isValidPageStepIncrement(this.settings.pageStepIncrement) ? this.settings.pageStepIncrement : null; + let pageStepCount = isValidPageStepCount(this.settings.pageStepCount) ? this.settings.pageStepCount : null; if (Number.isInteger(pageSize) && pageSize > 0) { this.defaultPageSize = pageSize; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.models.ts index eb944e3c65..1ff7a21d71 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.models.ts @@ -564,11 +564,18 @@ export function getHeaderTitle(dataKey: DataKey, keySettings: TableWidgetDataKey export function buildPageStepSizeValues(pageStepCount: number, pageStepIncrement: number): Array { const pageSteps: Array = []; - if (Number.isInteger(pageStepCount) && pageStepCount > 0 && pageStepCount <= 100 && - Number.isInteger(pageStepIncrement) && pageStepIncrement > 0) { + if (isValidPageStepCount(pageStepCount) && isValidPageStepIncrement(pageStepIncrement)) { for (let i = 1; i <= pageStepCount; i++) { pageSteps.push(pageStepIncrement * i); } } return pageSteps; } + +export function isValidPageStepIncrement(value: number): boolean { + return Number.isInteger(value) && value > 0; +} + +export function isValidPageStepCount(value: number): boolean { + return Number.isInteger(value) && value > 0 && value <= 100; +} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts index 010e49fe32..2e3a64c190 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/timeseries-table-widget.component.ts @@ -49,7 +49,6 @@ import { hashCode, isDefined, isDefinedAndNotNull, - isNumber, isObject, isUndefined } from '@core/utils'; @@ -85,6 +84,8 @@ import { getColumnSelectionAvailability, getRowStyleInfo, getTableCellButtonActions, + isValidPageStepCount, + isValidPageStepIncrement, noDataMessage, prepareTableCellButtonActions, RowStyleInfo, @@ -352,10 +353,8 @@ export class TimeseriesTableWidgetComponent extends PageComponent implements OnI this.rowStylesInfo = getRowStyleInfo(this.ctx, this.settings, 'rowData, ctx'); const pageSize = this.settings.defaultPageSize; - let pageStepIncrement = Number.isInteger(this.settings.pageStepIncrement) && this.settings.pageStepIncrement > 0 ? - this.settings.pageStepIncrement : null; - let pageStepCount = Number.isInteger(this.settings.pageStepCount) && this.settings.pageStepCount > 0 - && this.settings.pageStepCount <= 100 ? this.settings.pageStepCount : null; + let pageStepIncrement = isValidPageStepIncrement(this.settings.pageStepIncrement) ? this.settings.pageStepIncrement : null; + let pageStepCount = isValidPageStepCount(this.settings.pageStepCount) ? this.settings.pageStepCount : null; if (Number.isInteger(pageSize) && pageSize > 0) { this.defaultPageSize = pageSize; From 6f9e36d386cba656deb5a57114566d670a22e941 Mon Sep 17 00:00:00 2001 From: yevhenii Date: Thu, 24 Apr 2025 18:36:39 +0300 Subject: [PATCH 10/79] Update edge install instructions - updated install instructions --- .../install/centos/instructions.md | 144 ++++++++---------- .../install/docker/instructions.md | 61 +++++--- .../install/ubuntu/instructions.md | 112 +++++++------- 3 files changed, 153 insertions(+), 164 deletions(-) diff --git a/application/src/main/data/json/edge/instructions/install/centos/instructions.md b/application/src/main/data/json/edge/instructions/install/centos/instructions.md index 86c5acad56..90d6d6ddc8 100644 --- a/application/src/main/data/json/edge/instructions/install/centos/instructions.md +++ b/application/src/main/data/json/edge/instructions/install/centos/instructions.md @@ -4,15 +4,15 @@ Here is the list of commands, that can be used to quickly install ThingsBoard Ed Before continue to installation execute the following commands in order to install necessary tools: ```bash -sudo yum install -y nano wget -sudo yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm +sudo yum install -y nano wget && sudo yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm +{:copy-code} ``` -#### Install Java 17 (OpenJDK) +#### Step 1. Install Java 17 (OpenJDK) ThingsBoard service is running on Java 17. Follow these instructions to install OpenJDK 17: ```bash -sudo yum install java-17-openjdk +sudo dnf install java-17-openjdk {:copy-code} ``` @@ -39,112 +39,92 @@ OpenJDK Runtime Environment (...) OpenJDK 64-Bit Server VM (build ...) ``` -#### Configure PostgreSQL +#### Step 2. Configure ThingsBoard Database +ThingsBoard Edge supports SQL and hybrid database approaches. + +### PostgresSql ThingsBoard Edge uses PostgreSQL database as a local storage. -Instructions listed below will help you to install PostgreSQL. +To install PostgreSQL, follow the instructions below. ```bash # Update your system -sudo yum update +sudo dnf update {:copy-code} ``` -**For CentOS 7:** +Install the repository RPM: +**For CentOS/RHEL 8:** ```bash -# Install the repository RPM (for CentOS 7): -sudo yum -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm -# Install packages -sudo yum -y install epel-release yum-utils -sudo yum-config-manager --enable pgdg16 -sudo yum install postgresql16-server postgresql16 postgresql16-contrib -# Initialize your PostgreSQL DB -sudo /usr/pgsql-16/bin/postgresql-16-setup initdb -sudo systemctl start postgresql-16 -# Optional: Configure PostgreSQL to start on boot -sudo systemctl enable --now postgresql-16 - +# Install the repository RPM (For CentOS/RHEL 8): +sudo sudo dnf -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm {:copy-code} ``` -**For CentOS 8:** +**For CentOS/RHEL 9:** ```bash -# Install the repository RPM (for CentOS 8): -sudo yum -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm -# Install packages -sudo dnf -qy module disable postgresql -sudo dnf -y install postgresql16 postgresql16-server postgresql16-contrib -# Initialize your PostgreSQL DB -sudo /usr/pgsql-16/bin/postgresql-16-setup initdb -sudo systemctl start postgresql-16 -# Optional: Configure PostgreSQL to start on boot -sudo systemctl enable --now postgresql-16 - +# Install the repository RPM (for CentOS 9): +sudo dnf -y install https://download.postgresql.org/pub/repos/yum/reporpms/EL-9-x86_64/pgdg-redhat-repo-latest.noarch.rpm {:copy-code} ``` -Once PostgreSQL is installed you may want to create a new user or set the password for the main user. -The instructions below will help to set the password for main PostgreSQL user: +Install packages and initialize PostgreSQL. The PostgreSQL service will automatically start every time the system boots up. -```text -sudo su - postgres -psql -\password -\q +```bash +sudo dnf -qy module disable postgresql && \ +sudo dnf -y install postgresql16 postgresql16-server postgresql16-contrib && \ +sudo /usr/pgsql-16/bin/postgresql-16-setup initdb && \ +sudo systemctl enable --now postgresql-16 +{:copy-code} ``` -Then, press "Ctrl+D" to return to main user console. +Once PostgreSQL is installed, it is recommended to set the password for the PostgreSQL main user. -After configuring the password, edit the pg_hba.conf to use MD5 authentication with the postgres user. - -Edit pg_hba.conf file: +The following command will switch the current user to the PostgreSQL user and set the password directly in PostgreSQL. ```bash -sudo nano /var/lib/pgsql/16/data/pg_hba.conf +sudo -u postgres psql -c "\password" {:copy-code} ``` -Locate the following lines: - -```text -# IPv4 local connections: -host all all 127.0.0.1/32 ident -``` - -Replace `ident` with `md5`: +Then, enter and confirm the password. -```text -host all all 127.0.0.1/32 md5 -``` +Since ThingsBoard Edge uses the PostgreSQL database for local storage, configuring MD5 authentication ensures that only authenticated users or +applications can access the database, thus protecting your data. After configuring the password, +edit the pg_hba.conf file to use MD5 hashing for authentication instead of the default method (ident) for local IPv4 connections. -Finally, you should restart the PostgreSQL service to initialize the new configuration: +To replace ident with md5, run the following command: ```bash -sudo systemctl restart postgresql-16.service +sudo sed -i 's/^host\s\+all\s\+all\s\+127\.0\.0\.1\/32\s\+ident/host all all 127.0.0.1\/32 md5/' /var/lib/pgsql/16/data/pg_hba.conf {:copy-code} ``` -Connect to the database to create ThingsBoard Edge DB: +Then run the command that will restart the PostgreSQL service to apply configuration changes, connect to the database as a postgres user, +and create the ThingsBoard Edge database (tb_edge). To connect to the PostgreSQL database, enter the PostgreSQL password. ```bash -psql -U postgres -d postgres -h 127.0.0.1 -W +sudo systemctl restart postgresql-16.service && psql -U postgres -d postgres -h 127.0.0.1 -W -c "CREATE DATABASE tb_edge;" {:copy-code} ``` -Execute create database statement: +#### Step 3. Choose Queue Service -```bash -CREATE DATABASE tb_edge; -\q -{:copy-code} -``` +ThingsBoard Edge is able to use different messaging systems/brokers for storing the messages and communication between ThingsBoard services. +How to choose the right queue implementation? -#### ThingsBoard Edge service installation +In Memory queue implementation is built-in and default. It is useful for development(PoC) environments and is not suitable for production deployments or any sort of cluster deployments. + +Kafka is recommended for production deployments. This queue is used on the most of ThingsBoard production environments now. + +In Memory queue is built in and enabled by default. No additional configuration is required. + +#### Step 4. ThingsBoard Edge Service Installation Download installation package: ```bash -wget https://github.com/thingsboard/thingsboard-edge/releases/download/v${TB_EDGE_TAG}/tb-edge-${TB_EDGE_TAG}.rpm +wget wget https://github.com/thingsboard/thingsboard-edge/releases/download/v${TB_EDGE_TAG}/tb-edge-${TB_EDGE_TAG}.rpm {:copy-code} ``` @@ -155,7 +135,7 @@ sudo rpm -Uvh tb-edge-${TB_EDGE_TAG}.rpm {:copy-code} ``` -#### Configure ThingsBoard Edge +#### Step 5. Configure ThingsBoard Edge To configure ThingsBoard Edge, you can use the following command to automatically update the configuration file with specific values: ```bash @@ -169,25 +149,21 @@ EOL' {:copy-code} ``` -##### [Optional] Database Configuration -In case you changed default PostgreSQL datasource settings (**postgres**/**postgres**) please update the configuration file (**/etc/tb-edge/conf/tb-edge.conf**) with your actual values: - -```bash -sudo nano /etc/tb-edge/conf/tb-edge.conf -{:copy-code} -``` - -Please update the following lines in your configuration file. Make sure **to replace**: -- Replace 'postgres' with your actual PostgreSQL username; -- Replace 'PUT_YOUR_POSTGRESQL_PASSWORD_HERE' with your actual PostgreSQL password. +##### Configure PostgreSQL (Optional) +If you changed PostgreSQL default datasource settings, use the following command: ```bash +sudo sh -c 'cat <> /etc/tb-edge/conf/tb-edge.conf export SPRING_DATASOURCE_URL=jdbc:postgresql://localhost:5432/tb_edge export SPRING_DATASOURCE_USERNAME=postgres -export SPRING_DATASOURCE_PASSWORD=PUT_YOUR_POSTGRESQL_PASSWORD_HERE +export SPRING_DATASOURCE_PASSWORD= +EOL' {:copy-code} ``` +PUT_YOUR_POSTGRESQL_PASSWORD_HERE: Replace with your actual PostgreSQL user password. + + ##### [Optional] Update bind ports If ThingsBoard Edge is going to be running on the same machine where ThingsBoard server (cloud) is running, you'll need to update configuration parameters to avoid port collision between ThingsBoard server and ThingsBoard Edge. @@ -206,7 +182,7 @@ EOL' Make sure that ports above (18080, 11883, 15683) are not used by any other application. -#### Run installation script +#### Step 6. Run installation Script Once ThingsBoard Edge is installed and configured please execute the following install script: ```bash @@ -214,18 +190,18 @@ sudo /usr/share/tb-edge/bin/install/install.sh {:copy-code} ``` -#### Restart ThingsBoard Edge service +#### Step 7. Restart ThingsBoard Edge Service ```bash sudo service tb-edge restart {:copy-code} ``` -#### Open ThingsBoard Edge UI +#### Step 8. Open ThingsBoard Edge UI Once started, you will be able to open **ThingsBoard Edge UI** using the following link http://localhost:8080. ###### NOTE: Edge HTTP bind port update -Use next **ThingsBoard Edge UI** link **http://localhost:18080** if you updated HTTP 8080 bind port to **18080**. +If the Edge HTTP bind port was changed to 18080 during Edge installation, access the ThingsBoard Edge instance at http://localhost:18080. diff --git a/application/src/main/data/json/edge/instructions/install/docker/instructions.md b/application/src/main/data/json/edge/instructions/install/docker/instructions.md index 23f484f2ec..4457962d86 100644 --- a/application/src/main/data/json/edge/instructions/install/docker/instructions.md +++ b/application/src/main/data/json/edge/instructions/install/docker/instructions.md @@ -4,9 +4,26 @@ Here is the list of commands, that can be used to quickly install ThingsBoard Ed Install Docker CE and Docker Compose. -#### Running ThingsBoard Edge as docker service +#### Step 1. Running ThingsBoard Edge -Create docker compose file for ThingsBoard Edge service: +Here you can find ThingsBoard Edge docker image: + + thingsboard/tb-edge + +#### Step 2. Choose Queue and/or Database Services + +ThingsBoard Edge is able to use different messaging systems/brokers for storing the messages and communication between ThingsBoard services. +How to choose the right queue implementation? + +In Memory queue implementation is built-in and default. It is useful for development(PoC) environments and is not suitable for production deployments or any sort of cluster deployments. + +Kafka is recommended for production deployments. This queue is used on the most of ThingsBoard production environments now. + +Hybrid implementation combines PostgreSQL and Cassandra databases with Kafka queue service. It is recommended if you plan to manage 1M+ devices in production or handle high data ingestion rate (more than 5000 msg/sec). + +Create a docker compose file for the ThingsBoard Edge service: + +##### In Memory ```bash nano docker-compose.yml @@ -20,25 +37,22 @@ version: '3.8' services: mytbedge: restart: always - image: "thingsboard/tb-edge:${TB_EDGE_VERSION}" + image: "thingsboard/tb-edge:3.9.1EDGE" ports: - "8080:8080" - "1883:1883" - "5683-5688:5683-5688/udp" environment: SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/tb-edge - CLOUD_ROUTING_KEY: ${CLOUD_ROUTING_KEY} - CLOUD_ROUTING_SECRET: ${CLOUD_ROUTING_SECRET} - CLOUD_RPC_HOST: ${BASE_URL} - CLOUD_RPC_PORT: ${CLOUD_RPC_PORT} - CLOUD_RPC_SSL_ENABLED: ${CLOUD_RPC_SSL_ENABLED} + CLOUD_ROUTING_KEY: PUT_YOUR_EDGE_KEY_HERE # e.g. 19ea7ee8-5e6d-e642-4f32-05440a529015 + CLOUD_ROUTING_SECRET: PUT_YOUR_EDGE_SECRET_HERE # e.g. bztvkvfqsye7omv9uxlp + CLOUD_RPC_HOST: PUT_YOUR_CLOUD_IP # e.g. 192.168.1.1 or demo.thingsboard.io volumes: - tb-edge-data:/data - tb-edge-logs:/var/log/tb-edge - ${EXTRA_HOSTS} postgres: restart: always - image: "postgres:16" + image: "postgres:15" ports: - "5432" environment: @@ -58,24 +72,20 @@ volumes: ``` ##### [Optional] Update bind ports -If ThingsBoard Edge is going to be running on the same machine where ThingsBoard server (cloud) is running, you'll need to update docker compose port mapping to avoid port collision between ThingsBoard server and ThingsBoard Edge. +If ThingsBoard Edge is set to run on the same machine where the ThingsBoard server is operating, you need to update port configuration to prevent port collision between the ThingsBoard server and ThingsBoard Edge. -Please update next lines of `docker-compose.yml` file: +Ensure that the ports 18080, 11883, 15683-15688 are not used by any other application. -```text -ports: - - "18080:8080" - - "11883:1883" - - "15683-15688:5683-5688/udp" +Then, update the port configuration in the docker-compose.yml file: +```bash +sed -i ‘s/8080:8080/18080:8080/; s/1883:1883/11883:1883/; s/5683-5688:5683-5688\/udp/15683-15688:5683-5688\/udp/’ docker-compose.yml +{:copy-code} ``` -Make sure that ports above (18080, 11883, 15683-15688) are not used by any other application. - #### Start ThingsBoard Edge -Set the terminal in the directory which contains the `docker-compose.yml` file and execute the following commands to up this docker compose directly: +Set the terminal in the directory which contains the docker-compose.yml file and execute the following commands to up this docker compose directly: ```bash -docker compose up -d -docker compose logs -f mytbedge +docker compose up -d && docker compose logs -f mytbedge {:copy-code} ``` @@ -90,11 +100,12 @@ docker-compose up -d docker-compose logs -f mytbedge ``` -#### Open ThingsBoard Edge UI +#### Step 3. Open ThingsBoard Edge UI -Once started, you will be able to open **ThingsBoard Edge UI** using the following link http://localhost:8080. +Once the Edge service is started, open the Edge UI at http://localhost:8080. ###### NOTE: Edge HTTP bind port update -Use next **ThingsBoard Edge UI** link **http://localhost:18080** if you updated HTTP 8080 bind port to **18080**. +If the Edge HTTP bind port was changed to 18080 during Edge installation, access the ThingsBoard Edge instance at http://localhost:18080. +Please use your tenant credentials from local Server instance or ThingsBoard Live Demo to log in to the ThingsBoard Edge. diff --git a/application/src/main/data/json/edge/instructions/install/ubuntu/instructions.md b/application/src/main/data/json/edge/instructions/install/ubuntu/instructions.md index 992b5e2ee2..9d685c576a 100644 --- a/application/src/main/data/json/edge/instructions/install/ubuntu/instructions.md +++ b/application/src/main/data/json/edge/instructions/install/ubuntu/instructions.md @@ -1,51 +1,48 @@ Here is the list of commands, that can be used to quickly install ThingsBoard Edge on Ubuntu Server and connect to the server. -#### Install Java 17 (OpenJDK) -ThingsBoard service is running on Java 17. Follow these instructions to install OpenJDK 17: +#### Step 1. Install Java 17 (OpenJDK) +ThingsBoard service is running on Java 17. To install OpenJDK 17, follow these instructions: ```bash -sudo apt update -sudo apt install openjdk-17-jdk +sudo apt update && sudo apt install openjdk-17-jdk {:copy-code} ``` -Please don't forget to configure your operating system to use OpenJDK 17 by default. -You can configure which version is the default using the following command: +Configure your operating system to use OpenJDK 17 by default. You can configure the default version by running the following command: ```bash sudo update-alternatives --config java {:copy-code} ``` -You can check the installation using the following command: +To check the installed Java version on your system, use the following command: ```bash java -version {:copy-code} ``` -Expected command output is: +The expected result is: ```text -openjdk version "17.x.xx" +openjdk version "17.x.xx" OpenJDK Runtime Environment (...) -OpenJDK 64-Bit Server VM (build ...) +OpenJDK 64-Bit Server VM (...) ``` -#### Configure PostgreSQL -ThingsBoard Edge uses PostgreSQL database as a local storage. -Instructions listed below will help you to install PostgreSQL. +#### Step 2. Configure ThingsBoard Edge Database -```bash -# install **wget** if not already installed: -sudo apt install -y wget +ThingsBoard Edge supports SQL and hybrid database approaches. See the architecture page for details. -# import the repository signing key: -wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - +### Configure PostgreSQL +ThingsBoard Edge uses PostgreSQL database as a local storage. -# add repository contents to your system: -RELEASE=$(lsb_release -cs) -echo "deb http://apt.postgresql.org/pub/repos/apt/ ${RELEASE}"-pgdg main | sudo tee /etc/apt/sources.list.d/pgdg.list +To install the PostgreSQL database, run these commands: + +```bash +# Automated repository configuration: +sudo apt install -y postgresql-common +sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh # install and launch the postgresql service: sudo apt update @@ -54,25 +51,35 @@ sudo service postgresql start {:copy-code} ``` -Once PostgreSQL is installed you may want to create a new user or set the password for the main user. -The instructions below will help to set the password for main PostgreSQL user: +Once PostgreSQL is installed, it is recommended to set the password for the PostgreSQL main user. -```text -sudo su - postgres -psql -\password -\q +The following command will switch the current user to the PostgreSQL user and set the password directly in PostgreSQL. + +```bash +sudo -u postgres psql -c "\password" +{:copy-code} ``` -Then, press “Ctrl+D” to return to main user console and connect to the database to create ThingsBoard Edge DB: +Then, enter and confirm the password. -```text -psql -U postgres -d postgres -h 127.0.0.1 -W -CREATE DATABASE tb_edge; -\q +Finally, create a new PostgreSQL database named tb_edge by running the following command: + +```bash +echo "CREATE DATABASE tb_edge;" | psql -U postgres -d postgres -h 127.0.0.1 -W +{:copy-code} ``` -#### Thingsboard Edge service installation +#### Step 3. Choose Queue Service + +ThingsBoard Edge can use different messaging systems and brokers for storing messages and enabling communication between its services. Choose the appropriate queue implementation based on your specific business needs: + +In Memory: The built-in and default queue implementation. It is useful for development or proof-of-concept (PoC) environments, but is not recommended for production or any type of clustered deployments due to limited scalability. + +Kafka: Recommended for production deployments. This queue is used in the most of ThingsBoard production environments now. + +In Memory queue is built in and enabled by default. No additional configuration is required. + +#### Step 4. ThingsBoard Edge Service Installation Download installation package: ```bash @@ -87,39 +94,34 @@ sudo dpkg -i tb-edge-${TB_EDGE_TAG}.deb {:copy-code} ``` -#### Configure ThingsBoard Edge +#### Step 5. Configure ThingsBoard Edge To configure ThingsBoard Edge, you can use the following command to automatically update the configuration file with specific values: ```bash sudo sh -c 'cat <> /etc/tb-edge/conf/tb-edge.conf -export CLOUD_ROUTING_KEY=${CLOUD_ROUTING_KEY} -export CLOUD_ROUTING_SECRET=${CLOUD_ROUTING_SECRET} -export CLOUD_RPC_HOST=${BASE_URL} -export CLOUD_RPC_PORT=${CLOUD_RPC_PORT} -export CLOUD_RPC_SSL_ENABLED=${CLOUD_RPC_SSL_ENABLED} +export CLOUD_ROUTING_KEY= +export CLOUD_ROUTING_SECRET= +export CLOUD_RPC_HOST=demo.thingsboard.io +export CLOUD_RPC_PORT=7070 +export CLOUD_RPC_SSL_ENABLED=false EOL' {:copy-code} ``` -##### [Optional] Database Configuration -In case you changed default PostgreSQL datasource settings (**postgres**/**postgres**) please update the configuration file (**/etc/tb-edge/conf/tb-edge.conf**) with your actual values: - -```bash -sudo nano /etc/tb-edge/conf/tb-edge.conf -{:copy-code} -``` - -Please update the following lines in your configuration file. Make sure **to replace**: -- Replace 'postgres' with your actual PostgreSQL username; -- Replace 'PUT_YOUR_POSTGRESQL_PASSWORD_HERE' with your actual PostgreSQL password. +##### [Optional] Configure PostgreSQL +If you changed PostgreSQL default datasource settings, use the following command: ```bash +sudo sh -c 'cat <> /etc/tb-edge/conf/tb-edge.conf export SPRING_DATASOURCE_URL=jdbc:postgresql://localhost:5432/tb_edge export SPRING_DATASOURCE_USERNAME=postgres -export SPRING_DATASOURCE_PASSWORD=PUT_YOUR_POSTGRESQL_PASSWORD_HERE +export SPRING_DATASOURCE_PASSWORD= +EOL' {:copy-code} ``` +PUT_YOUR_POSTGRESQL_PASSWORD_HERE: Replace with your actual PostgreSQL user password. + ##### [Optional] Update bind ports If ThingsBoard Edge is going to be running on the same machine where ThingsBoard server (cloud) is running, you'll need to update configuration parameters to avoid port collision between ThingsBoard server and ThingsBoard Edge. @@ -138,7 +140,7 @@ EOL' Make sure that ports above (18080, 11883, 15683) are not used by any other application. -#### Run installation script +#### Step 6. Run installation Script Once ThingsBoard Edge is installed and configured please execute the following install script: @@ -147,14 +149,14 @@ sudo /usr/share/tb-edge/bin/install/install.sh {:copy-code} ``` -#### Restart ThingsBoard Edge service +#### Step 7. Restart ThingsBoard Edge Service ```bash sudo service tb-edge restart {:copy-code} ``` -#### Open ThingsBoard Edge UI +#### Step 8. Open ThingsBoard Edge UI Once started, you will be able to open **ThingsBoard Edge UI** using the following link http://localhost:8080. From c158795041a41ed0ac01d92f0d054d5896ec094a Mon Sep 17 00:00:00 2001 From: Yevhenii Date: Fri, 25 Apr 2025 20:32:23 +0300 Subject: [PATCH 11/79] Changed EdgeGrpcService startup - Refactored init method to use @AfterStartUp with REGULAR_SERVICE order --- .../server/service/edge/rpc/EdgeGrpcService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java index 5f137848b1..73da3694f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java @@ -22,7 +22,6 @@ import io.grpc.Server; import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; import io.grpc.stub.StreamObserver; import jakarta.annotation.Nullable; -import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -63,6 +62,7 @@ import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; +import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.EdgeContextComponent; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; @@ -162,8 +162,8 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i private ScheduledExecutorService executorService; - @PostConstruct - public void init() { + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void onStartUp() { log.info("Initializing Edge RPC service!"); NettyServerBuilder builder = NettyServerBuilder.forPort(rpcPort) .permitKeepAliveTime(clientMaxKeepAliveTimeSec, TimeUnit.SECONDS) From 8c178a26477bd3c6960143586441804da617f42a Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 30 Apr 2025 16:46:12 +0300 Subject: [PATCH 12/79] UI: Fixed incorrect help links for calculated fields --- ui-ngx/src/app/shared/models/constants.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/shared/models/constants.ts b/ui-ngx/src/app/shared/models/constants.ts index e1306e577c..af576882b8 100644 --- a/ui-ngx/src/app/shared/models/constants.ts +++ b/ui-ngx/src/app/shared/models/constants.ts @@ -198,7 +198,7 @@ export const HelpLinks = { mobileApplication: `${helpBaseUrl}/docs${docPlatformPrefix}/mobile-center/applications/`, mobileBundle: `${helpBaseUrl}/docs${docPlatformPrefix}/mobile-center/mobile-center/`, mobileQrCode: `${helpBaseUrl}/docs${docPlatformPrefix}/user-guide/ui/mobile-qr-code/`, - calculatedField: `${helpBaseUrl}/docs${docPlatformPrefix}/`, + calculatedField: `${helpBaseUrl}/docs${docPlatformPrefix}/user-guide/calculated-fields/`, timewindowSettings: `${helpBaseUrl}/docs${docPlatformPrefix}/user-guide/dashboards/#time-window`, } }; From ff3c1e27ed83336a9d3d46b9c748d4ae562993a7 Mon Sep 17 00:00:00 2001 From: Vladyslav Prykhodko Date: Wed, 30 Apr 2025 23:47:19 +0300 Subject: [PATCH 13/79] =?UTF-8?q?UI:=20Improved=20Nederlands=20(Belgi?= =?UTF-8?q?=C3=AB)=20translation=20remove=20duplicate=20tranlate=20and=20i?= =?UTF-8?q?nvalid=20text?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../assets/locale/locale.constant-nl_BE.json | 34 +++---------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/ui-ngx/src/assets/locale/locale.constant-nl_BE.json b/ui-ngx/src/assets/locale/locale.constant-nl_BE.json index 348eea95b0..e64970bca3 100644 --- a/ui-ngx/src/assets/locale/locale.constant-nl_BE.json +++ b/ui-ngx/src/assets/locale/locale.constant-nl_BE.json @@ -59,8 +59,6 @@ "import": "Importeren", "export": "Exporteren", "share-via": "Delen via {{provider}}", - "move": "Verplaatsen", - "select": "Selecteren", "continue": "Voortzetten", "discard-changes": "Wijzigingen negeren", "download": "Downloaden", @@ -291,7 +289,6 @@ "enable": "OAuth2-instellingen inschakelen", "domains": "Domeinen", "mobile-apps": "Mobiele applicaties", - "no-mobile-apps": "Geen applicaties geconfigureerd", "mobile-package": "Applicatie pakket", "mobile-package-placeholder": "Vb.: my.example.app", "mobile-package-hint": "Voor Android: uw eigen unieke applicatie-ID. Voor iOS: ID van productbundel.", @@ -299,7 +296,6 @@ "mobile-app-secret": "Geheim van de toepassing", "invalid-mobile-app-secret": "Het geheim van de toepassing mag alleen alfanumerieke tekens bevatten en moet tussen de 16 en 2048 tekens lang zijn.", "copy-mobile-app-secret": "Toepassingsgeheim kopiëren", - "add-mobile-app": "Applicatie toevoegen", "delete-mobile-app": "Toepassingsgegevens verwijderen", "providers": "Providers", "platform-web": "Web", @@ -627,8 +623,6 @@ "filter-type-entity-view-search-query": "Zoekquery voor entiteitsweergave", "filter-type-entity-view-search-query-description": "Entiteitsweergaven met types {{entityViewTypes}} die {{relationType}} relatie hebben {{direction}} {{rootEntity}}", "filter-type-apiUsageState": "Api-gebruiksstatus", - "filter-type-edge-search-query": "Edge-zoekopdracht", - "filter-type-edge-search-query-description": "Edge met types {{edgeTypes}} die {{relationType}} relatie hebben {{direction}} {{rootEntity}}", "entity-filter": "Entiteit filteren", "resolve-multiple": "Oplossen als meerdere entiteiten", "resolve-multiple-hint": "Inschakelen om gegevens van alle gefilterde entiteiten tegelijk weer te geven. \nAls uitgeschakeld, toont de widget alleen gegevens van de geselecteerde entiteit.", @@ -727,7 +721,6 @@ "asset-required": "Asset is vereist", "name-starts-with": "Expressie van itemnaam", "help-text": "Gebruik '%' naar behoefte: '%asset_name_contains%', '%asset_name_ends', 'asset_starts_with'.", - "search": "Assets zoeken", "select-group-to-add": "Selecteer doelgroep om geselecteerde assets toe te voegen", "select-group-to-move": "Selecteer de doelgroep om geselecteerde assets te verplaatsen", "remove-assets-from-group": "Weet je zeker dat je { count, plural, =1 {1 asset} other {# assets} } uit groep '{{entityGroup}}' wilt verwijderen?", @@ -804,7 +797,6 @@ "email-messages": "E-mailberichten", "email-messages-daily-activity": "Dagelijkse activiteit e-mailberichten", "email-messages-monthly-activity": "Maandelijkse activiteit e-mailberichten", - "exceptions": "Uitzonderingen", "executions": "Executies", "javascript": "JavaScript", "javascript-executions": "JavaScript-uitvoeringen", @@ -820,9 +812,7 @@ "permanent-failures": "${entityName} Permanente storingen", "permanent-timeouts": "Permanente time-outs van $ {entityName}", "processing-failures": "${entityName} Verwerkingsfouten", - "processing-failures-and-timeouts": "Verwerkingsfouten en time-outs", "processing-timeouts": "${entityName} time-outs voor verwerking", - "queue-stats": "Queue Statistieken", "rule-chain": "Rule chain", "rule-engine": "Rule engine", "rule-engine-daily-activity": "Dagelijkse activiteit van de rule engine", @@ -969,7 +959,7 @@ "name": "Naam", "name-required": "Naam is verplicht.", "name-max-length": "Naam moet kleiner zijn dan 256 tekens", - "description": "Omschrijving: __________", + "description": "Omschrijving", "decoder": "Decoder", "encoder": "Coderingsprogramma", "test-decoder-fuction": "Test decoder functie", @@ -1047,7 +1037,6 @@ "manage-customer-assets": "Klant devices beheren", "manage-public-assets": "Openbare asset beheren", "manage-customer-edges": "Beheer klant edges", - "manage-public-assets": "Openbare asset beheren", "add-customer-text": "Nieuwe klant toevoegen", "no-customers-text": "Geen klanten gevonden", "customer-details": "Klantgegevens", @@ -2451,7 +2440,7 @@ "name": "Naam", "name-required": "Naam is verplicht.", "name-max-length": "Naam moet kleiner zijn dan 256 tekens", - "description": "Omschrijving: __________", + "description": "Omschrijving", "add": "Entiteitsgroep toevoegen", "open-entity-group": "Entiteitsgroep openen", "add-entity-group-text": "Nieuwe entiteitsgroep toevoegen", @@ -2655,8 +2644,6 @@ "assign-entity-views": "Entiteitsweergaven toewijzen", "assign-entity-views-text": "Wijs { count, plural, =1 {1 entity view} other {# entity views} } toe aan de klant", "delete-entity-views": "Entiteitsweergaven verwijderen", - "make-public": "Entiteitsweergave openbaar maken", - "make-private": "Entiteitsweergave privé maken", "unassign-from-customer": "Toewijzing van klant ongedaan maken", "unassign-entity-views": "Toewijzing van entiteitsweergaven intrekken", "unassign-entity-views-action-title": "Toewijzing { count, plural, =1 {1 entity view} other {# entity views} } van klant ongedaan maken", @@ -2666,10 +2653,6 @@ "delete-entity-views-title": "Weet u zeker dat u { count, plural, =1 {1 entity view} other {# entity views} } wilt verwijderen?", "delete-entity-views-action-title": "Verwijder { count, plural, =1 {1 entity view} other {# entity views} }", "delete-entity-views-text": "Opgelet, na de bevestiging worden alle geselecteerde entiteitsweergaven verwijderd en kunnen alle gerelateerde gegevens niet meer worden hersteld.", - "make-public-entity-view-title": "Weet u zeker dat u de entiteitsweergave '{{entityViewName}}' openbaar wilt maken?", - "make-public-entity-view-text": "Na de bevestiging worden de entiteitsweergave en al haar gegevens openbaar en toegankelijk gemaakt voor anderen.", - "make-private-entity-view-title": "Weet u zeker dat u de entiteitsweergave '{{entityViewName}}' privé wilt maken?", - "make-private-entity-view-text": "Na de bevestiging worden de entiteitsweergave en al zijn gegevens privé gemaakt en zijn ze niet toegankelijk voor anderen.", "unassign-entity-view-title": "Weet u zeker dat u de toewijzing van de entiteitsweergave '{{entityViewName}}' wilt opheffen?", "unassign-entity-view-text": "Na de bevestiging wordt de toewijzing van de entiteitsweergave ongedaan gemaakt en is deze niet toegankelijk voor de klant.", "unassign-entity-view": "Toewijzing van entiteitsweergave ongedaan maken", @@ -2760,7 +2743,6 @@ "type": "Type", "in": "In", "out": "Buiten", - "metadata": "Metagegevens", "message": "Bericht", "entity": "Entiteit", "message-id": "Bericht-ID", @@ -3319,7 +3301,7 @@ "name": "Naam", "name-required": "Naam is verplicht.", "name-max-length": "Naam moet kleiner zijn dan 256 tekens", - "description": "Omschrijving: __________", + "description": "Omschrijving", "base-url": "Basis-URL", "base-url-required": "Basis-URL is vereist", "security-key": "Beveiligingssleutel", @@ -4596,7 +4578,7 @@ "name": "Naam", "name-required": "Naam is verplicht.", "name-max-length": "Naam moet kleiner zijn dan 256 tekens", - "description": "Omschrijving: __________", + "description": "Omschrijving", "events": "Events", "details": "Details", "copyId": "Rol-ID kopiëren", @@ -4641,7 +4623,7 @@ "permissions-required": "Er moet ten minste één machtigingsvermelding worden opgegeven.", "remove-permission": "Machtigingsinvoer verwijderen", "add-permission": "Machtigingsinvoer toevoegen", - "other": "Anders __________", + "other": "Anders", "resource": { "resource": "Hulpbron", "select-resource": "Bron selecteren", @@ -5781,12 +5763,6 @@ "delete-solution-text": "Opgelet, na de bevestiging worden de oplossing en alle gerelateerde gegevens onherstelbaar.", "installing": "Oplossingssjabloon installeren..." }, - "markdown": { - "edit": "Bewerken", - "preview": "Voorbeeld", - "copy-code": "Klik om te kopiëren", - "copied": "Gekopieerd!" - }, "white-labeling": { "white-labeling": "White labelling", "white-labeling-general": "Algemene White Labeling", From 3066d172944a72cf717f52ada4a2f5640a04b6c1 Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Thu, 1 May 2025 08:47:12 +0300 Subject: [PATCH 14/79] UI: Hint for dynamic settings and refactoring flow animation --- .../bottom-right-elbow-connector-hp.svg | 18 +- .../scada_symbols/bottom-tee-connector-hp.svg | 16 +- .../scada_symbols/cross-connector-hp.svg | 16 +- .../scada_symbols/horizontal-connector-hp.svg | 19 +- .../left-bottom-elbow-connector-hp.svg | 23 ++- .../scada_symbols/left-tee-connector-hp.svg | 18 +- .../left-top-elbow-connector-hp.svg | 18 +- .../long-horizontal-connector-hp.svg | 25 ++- .../long-vertical-connector-hp.svg | 22 ++- .../scada_symbols/right-tee-connector-hp.svg | 16 +- .../top-right-elbow-connector-hp.svg | 18 +- .../scada_symbols/top-tee-connector-hp.svg | 16 +- .../scada_symbols/vertical-connector-hp.svg | 22 ++- .../widget/lib/scada/scada-symbol.models.ts | 171 ++++++++++++++++++ ...dynamic-form-property-panel.component.html | 6 + .../dynamic-form-property-panel.component.ts | 1 + .../dynamic-form/dynamic-form.component.html | 10 +- .../app/shared/models/dynamic-form.models.ts | 3 + .../assets/locale/locale.constant-en_US.json | 1 + 19 files changed, 357 insertions(+), 82 deletions(-) diff --git a/application/src/main/data/json/system/scada_symbols/bottom-right-elbow-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/bottom-right-elbow-connector-hp.svg index fa273dc8ec..b76f7399cc 100644 --- a/application/src/main/data/json/system/scada_symbols/bottom-right-elbow-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/bottom-right-elbow-connector-hp.svg @@ -3,7 +3,7 @@ "description": "Bottom right elbow connector", "widgetSizeX": 1, "widgetSizeY": 1, - "stateRenderFunction": "const {\n flowAnimation,\n animationDirection: flowDirection,\n flowAnimationSpeed\n} = ctx.values;\nconst {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\nconst line = ctx.tags.line[0].attr('d');\nconst animation = ctx.tags.animationGroup[0];\nconst offset = Date.now() % 1000;\nconst duration = 1 / flowAnimationSpeed;\n\nconst prevFlowAnimation = animation.remember('flowAnimation');\nconst prevFlowDirection = animation.remember('flowDirection');\nconst prevFlowDuration = animation.remember('flowDuration');\n\nif (flowAnimation && flowAnimation !== prevFlowAnimation) {\n animation.remember('flowAnimation', flowAnimation);\n animation.remember('flowDuration', duration);\n animation.remember('flowDirection', flowDirection);\n animateFlow(offset, flowDirection);\n} else if (flowAnimation && flowDirection !== prevFlowDirection) {\n animation.remember('flowDirection', flowDirection);\n animateFlow(offset, flowDirection);\n} else if (flowAnimation && duration !== prevFlowDuration) {\n animation.remember('flowDuration', duration);\n animation.findOne('animate').attr('dur', `${duration}s`) ;\n} else if (!flowAnimation && prevFlowAnimation) {\n animation.remember('flowAnimation', null);\n animation.clear();\n}\n\nfunction animateFlow(offset, flowDirection) {\n animation.clear();\n const dashArray = `${dashWidth}${dashGap ? ` ${dashGap}` : ''}`;\n const value = flowDirection ? `-${dashWidth + (dashGap || dashWidth)}` : `${dashWidth + (dashGap || dashWidth)}`;\n\n animation.add(``);\n}\n", + "stateRenderFunction": "const {\n flowAnimation,\n animationDirection: flowDirection,\n flowAnimationSpeed\n} = ctx.values;\nconst {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\nconst line = ctx.tags.line[0].attr('d');\nconst lineReversed = 'M 200,100 L 125,100 Q 100,100 100,125 L 100, 200';\nconst animation = ctx.tags.animationGroup[0];\nconst duration = 1 / flowAnimationSpeed;\n\nlet animateFlow = ctx.api.connectorAnimation(animation);\n\nif (flowAnimation) {\n if (!animateFlow) {\n animateFlow = ctx.api.connectorAnimate(animation, line, lineReversed).flowAppearance(lineWidth, lineColor, dashCap, dashWidth, dashGap).duration(duration).direction(flowDirection).play();\n } else {\n animateFlow.duration(duration).direction(flowDirection).play();\n }\n} else {\n if (animateFlow) {\n animateFlow.finish();\n }\n}\n", "tags": [ { "tag": "line", @@ -140,13 +140,14 @@ }, { "id": "mainLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 6, "required": true, "subLabel": "Main", "divider": true, "fieldSuffix": "px", + "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, @@ -155,21 +156,23 @@ }, { "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 2, "required": true, "subLabel": "Secondary", + "divider": true, "fieldSuffix": "px", + "condition": "return !model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, - "visible": true + "visible": false }, { "id": "lineColor", - "name": "{i18n:scada.symbol.line-color}", + "name": "{i18n:scada.symbol.main-line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -182,6 +185,7 @@ "type": "number", "default": 4, "subLabel": "Width", + "divider": true, "fieldSuffix": "px", "min": 1, "step": 1, @@ -200,6 +204,7 @@ { "id": "flowStyleDash", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, @@ -215,6 +220,7 @@ { "id": "flowStyleGap", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, @@ -250,5 +256,5 @@ } ] }]]> - + \ No newline at end of file diff --git a/application/src/main/data/json/system/scada_symbols/bottom-tee-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/bottom-tee-connector-hp.svg index 2feb95e9ff..28e0788d35 100644 --- a/application/src/main/data/json/system/scada_symbols/bottom-tee-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/bottom-tee-connector-hp.svg @@ -3,7 +3,7 @@ "description": "Bottom tee connector", "widgetSizeX": 1, "widgetSizeY": 1, - "stateRenderFunction": "const {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\n\nconst leftLine = \"M0 100H100\";\nconst rightLine = \"M100 100H200\";\nconst bottomLine = \"M 100,200 V 103\";\n\nprepareFlowAnimation('left', leftLine);\nprepareFlowAnimation('right', rightLine);\nprepareFlowAnimation('bottom', bottomLine);\n\nfunction prepareFlowAnimation(prefix, line) {\n const flowAnimation = ctx.values[prefix + 'Flow'];\n const flowDirection = ctx.values[prefix + 'FlowDirection'];\n const flowAnimationSpeed = ctx.values[prefix + 'FlowAnimationSpeed'];\n\n const animation = ctx.tags[prefix + 'Line'][0];\n const offset = Date.now() % 1000;\n const duration = 1 / flowAnimationSpeed;\n \n const prevFlowAnimation = animation.remember('flowAnimation');\n const prevFlowDirection = animation.remember('flowDirection');\n const prevFlowDuration = animation.remember('flowDuration');\n \n if (flowAnimation && flowAnimation !== prevFlowAnimation) {\n animation.remember('flowAnimation', flowAnimation);\n animation.remember('flowDuration', duration);\n animation.remember('flowDirection', flowDirection);\n animateFlow(animation, offset, flowDirection, duration, line);\n } else if (flowAnimation && flowDirection !== prevFlowDirection) {\n animation.remember('flowDirection', flowDirection);\n animateFlow(animation, offset, flowDirection, duration, line);\n } else if (flowAnimation && duration !== prevFlowDuration) {\n animation.remember('flowDuration', duration);\n animation.findOne('animate').attr('dur', `${duration}s`) ;\n } else if (!flowAnimation && prevFlowAnimation) {\n animation.remember('flowAnimation', null);\n animation.clear();\n }\n}\n\nfunction animateFlow(group, offset, flowDirection, duration, line) {\n group.clear();\n const dashArray = `${dashWidth}${dashGap ? ` ${dashGap}` : ''}`;\n const value = flowDirection ? `${dashWidth + (dashGap || dashWidth)}` : `-${dashWidth + (dashGap || dashWidth)}`;\n\n group.add(``);\n}", + "stateRenderFunction": "const {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\n\nconst leftLine = \"M0 100H100\";\nconst leftLineReversed = \"M 100,100 H 0\";\nconst rightLine = \"M100 100H200\";\nconst rightLineReversed = \"M 200,100 H 100\";\nconst bottomLine = \"M 100,200 V 103\";\nconst bottomLineReversed = \"M 100,103 V 200\";\n\nprepareFlowAnimation('left', leftLine, leftLineReversed);\nprepareFlowAnimation('right', rightLine, rightLineReversed);\nprepareFlowAnimation('bottom', bottomLine, bottomLineReversed);\n\nfunction prepareFlowAnimation(prefix, line, reversedLine) {\n const flowAnimation = ctx.values[prefix + 'Flow'];\n const flowDirection = ctx.values[prefix + 'FlowDirection'];\n const flowAnimationSpeed = ctx.values[prefix + 'FlowAnimationSpeed'];\n\n const animation = ctx.tags[prefix + 'Line'][0];\n const duration = 1 / flowAnimationSpeed;\n \n let animateFlow = ctx.api.connectorAnimation(animation);\n \n if (flowAnimation) {\n if (!animateFlow) {\n animateFlow = ctx.api.connectorAnimate(animation, line, reversedLine).flowAppearance(lineWidth, lineColor, dashCap, dashWidth, dashGap).duration(duration).direction(flowDirection).play();\n } else {\n animateFlow.duration(duration).direction(flowDirection).play();\n }\n } else {\n if (animateFlow) {\n animateFlow.finish();\n }\n }\n}\n", "tags": [ { "tag": "line", @@ -377,13 +377,14 @@ }, { "id": "mainLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 6, "required": true, "subLabel": "Main", "divider": true, "fieldSuffix": "px", + "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, @@ -392,21 +393,23 @@ }, { "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 2, "required": true, "subLabel": "Secondary", + "divider": true, "fieldSuffix": "px", + "condition": "return !model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, - "visible": true + "visible": false }, { "id": "lineColor", - "name": "{i18n:scada.symbol.line-color}", + "name": "{i18n:scada.symbol.main-line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -419,6 +422,7 @@ "type": "number", "default": 4, "subLabel": "Width", + "divider": true, "fieldSuffix": "px", "min": 1, "step": 1, @@ -437,6 +441,7 @@ { "id": "flowStyleDash", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, @@ -452,6 +457,7 @@ { "id": "flowStyleGap", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, diff --git a/application/src/main/data/json/system/scada_symbols/cross-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/cross-connector-hp.svg index ab6798b48b..480666a589 100644 --- a/application/src/main/data/json/system/scada_symbols/cross-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/cross-connector-hp.svg @@ -3,7 +3,7 @@ "description": "Cross connector", "widgetSizeX": 1, "widgetSizeY": 1, - "stateRenderFunction": "const {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\n\nconst leftLine = \"M0 100H100\";\nconst topLine = \"M100 97L100 0\";\nconst rightLine = \"M100 100H200\";\nconst bottomLine = \"M 100,200 V 103\";\n\nprepareFlowAnimation('left', leftLine);\nprepareFlowAnimation('top', topLine);\nprepareFlowAnimation('right', rightLine);\nprepareFlowAnimation('bottom', bottomLine);\n\nfunction prepareFlowAnimation(prefix, line) {\n const flowAnimation = ctx.values[prefix + 'Flow'];\n const flowDirection = ctx.values[prefix + 'FlowDirection'];\n const flowAnimationSpeed = ctx.values[prefix + 'FlowAnimationSpeed'];\n\n const animation = ctx.tags[prefix + 'Line'][0];\n const offset = Date.now() % 1000;\n const duration = 1 / flowAnimationSpeed;\n \n const prevFlowAnimation = animation.remember('flowAnimation');\n const prevFlowDirection = animation.remember('flowDirection');\n const prevFlowDuration = animation.remember('flowDuration');\n \n if (flowAnimation && flowAnimation !== prevFlowAnimation) {\n animation.remember('flowAnimation', flowAnimation);\n animation.remember('flowDuration', duration);\n animation.remember('flowDirection', flowDirection);\n animateFlow(animation, offset, flowDirection, duration, line);\n } else if (flowAnimation && flowDirection !== prevFlowDirection) {\n animation.remember('flowDirection', flowDirection);\n animateFlow(animation, offset, flowDirection, duration, line);\n } else if (flowAnimation && duration !== prevFlowDuration) {\n animation.remember('flowDuration', duration);\n animation.findOne('animate').attr('dur', `${duration}s`) ;\n } else if (!flowAnimation && prevFlowAnimation) {\n animation.remember('flowAnimation', null);\n animation.clear();\n }\n}\n\nfunction animateFlow(group, offset, flowDirection, duration, line) {\n group.clear();\n const dashArray = `${dashWidth}${dashGap ? ` ${dashGap}` : ''}`;\n const value = flowDirection ? `${dashWidth + (dashGap || dashWidth)}` : `-${dashWidth + (dashGap || dashWidth)}`;\n\n group.add(``);\n}", + "stateRenderFunction": "const {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\n\nconst leftLine = \"M0 100H100\";\nconst leftLineReversed = \"M 100,100 H 0\";\nconst topLine = \"M100 97L100 0\";\nconst topLineReversed = \"M 100,0 V 97\";\nconst rightLine = \"M100 100H200\";\nconst rightLineReversed = \"M 200,100 H 100\";\nconst bottomLine = \"M 100,200 V 103\";\nconst bottomLineReversed = \"M 100,103 V 200\";\n\nprepareFlowAnimation('left', leftLine, leftLineReversed);\nprepareFlowAnimation('top', topLine, topLineReversed);\nprepareFlowAnimation('right', rightLine, rightLineReversed);\nprepareFlowAnimation('bottom', bottomLine, bottomLineReversed);\n\nfunction prepareFlowAnimation(prefix, line, reversedLine) {\n const flowAnimation = ctx.values[prefix + 'Flow'];\n const flowDirection = ctx.values[prefix + 'FlowDirection'];\n const flowAnimationSpeed = ctx.values[prefix + 'FlowAnimationSpeed'];\n\n const animation = ctx.tags[prefix + 'Line'][0];\n const duration = 1 / flowAnimationSpeed;\n \n let animateFlow = ctx.api.connectorAnimation(animation);\n \n if (flowAnimation) {\n if (!animateFlow) {\n animateFlow = ctx.api.connectorAnimate(animation, line, reversedLine).flowAppearance(lineWidth, lineColor, dashCap, dashWidth, dashGap).duration(duration).direction(flowDirection).play();\n } else {\n animateFlow.duration(duration).direction(flowDirection).play();\n }\n } else {\n if (animateFlow) {\n animateFlow.finish();\n }\n }\n}", "tags": [ { "tag": "line", @@ -493,13 +493,14 @@ }, { "id": "mainLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 6, "required": true, "subLabel": "Main", "divider": true, "fieldSuffix": "px", + "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, @@ -508,21 +509,23 @@ }, { "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 2, "required": true, "subLabel": "Secondary", + "divider": true, "fieldSuffix": "px", + "condition": "return !model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, - "visible": true + "visible": false }, { "id": "lineColor", - "name": "{i18n:scada.symbol.line-color}", + "name": "{i18n:scada.symbol.main-line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -535,6 +538,7 @@ "type": "number", "default": 4, "subLabel": "Width", + "divider": true, "fieldSuffix": "px", "min": 1, "step": 1, @@ -553,6 +557,7 @@ { "id": "flowStyleDash", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, @@ -568,6 +573,7 @@ { "id": "flowStyleGap", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, diff --git a/application/src/main/data/json/system/scada_symbols/horizontal-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/horizontal-connector-hp.svg index 74d048e884..230c7b68a5 100644 --- a/application/src/main/data/json/system/scada_symbols/horizontal-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/horizontal-connector-hp.svg @@ -3,7 +3,7 @@ "description": "Horizontal connector with an optional directional arrow to visually indicate flow.", "widgetSizeX": 1, "widgetSizeY": 1, - "stateRenderFunction": "const {\n flowAnimation,\n arrowDirection: flowDirection,\n flowAnimationSpeed\n} = ctx.values;\nconst {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\nconst line = ctx.tags.line[0].attr('d');\nconst animation = ctx.tags.animationGroup[0];\nconst offset = Date.now() % 1000;\nconst duration = 1 / flowAnimationSpeed;\n\nconst prevFlowAnimation = animation.remember('flowAnimation');\nconst prevFlowDirection = animation.remember('flowDirection');\nconst prevFlowDuration = animation.remember('flowDuration');\n\nif (flowAnimation && flowAnimation !== prevFlowAnimation) {\n animation.remember('flowAnimation', flowAnimation);\n animation.remember('flowDuration', duration);\n animation.remember('flowDirection', flowDirection);\n animateFlow(offset, flowDirection);\n} else if (flowAnimation && flowDirection !== prevFlowDirection) {\n animation.remember('flowDirection', flowDirection);\n animateFlow(offset, flowDirection);\n} else if (flowAnimation && duration !== prevFlowDuration) {\n animation.remember('flowDuration', duration);\n animation.findOne('animate').attr('dur', `${duration}s`) ;\n} else if (!flowAnimation && prevFlowAnimation) {\n animation.remember('flowAnimation', null);\n animation.clear();\n}\n\nfunction animateFlow(offset, flowDirection) {\n animation.clear();\n const dashArray = `${dashWidth}${dashGap ? ` ${dashGap}` : ''}`;\n const value = flowDirection ? `${dashWidth + (dashGap || dashWidth)}` : `-${dashWidth + (dashGap || dashWidth)}`;\n\n animation.add(``);\n}\n", + "stateRenderFunction": "const {\n flowAnimation,\n arrowDirection: flowDirection,\n flowAnimationSpeed\n} = ctx.values;\nconst {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\nconst line = ctx.tags.line[0].attr('d');\nconst lineReversed = 'M 200,100 H 0';\nconst animation = ctx.tags.animationGroup[0];\nconst duration = 1 / flowAnimationSpeed;\n\nlet animateFlow = ctx.api.connectorAnimation(animation);\n\nif (flowAnimation) {\n if (!animateFlow) {\n animateFlow = ctx.api.connectorAnimate(animation, line, lineReversed).flowAppearance(lineWidth, lineColor, dashCap, dashWidth, dashGap).duration(duration).direction(flowDirection).play();\n } else {\n animateFlow.duration(duration).direction(flowDirection).play();\n }\n} else {\n if (animateFlow) {\n animateFlow.finish();\n }\n}\n\n", "tags": [ { "tag": "arrow", @@ -176,13 +176,14 @@ }, { "id": "mainLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 6, "required": true, "subLabel": "Main", "divider": true, "fieldSuffix": "px", + "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, @@ -191,21 +192,23 @@ }, { "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 2, "required": true, "subLabel": "Secondary", + "divider": true, "fieldSuffix": "px", + "condition": "return !model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, - "visible": true + "visible": false }, { "id": "lineColor", - "name": "{i18n:scada.symbol.line-color}", + "name": "{i18n:scada.symbol.main-line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -229,11 +232,14 @@ "name": "{i18n:scada.symbol.flow}", "group": "{i18n:scada.symbol.animation}", "type": "color", - "default": "#C8DFF7" + "default": "#C8DFF7", + "disabled": false, + "visible": true }, { "id": "flowStyleDash", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, @@ -249,6 +255,7 @@ { "id": "flowStyleGap", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, diff --git a/application/src/main/data/json/system/scada_symbols/left-bottom-elbow-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/left-bottom-elbow-connector-hp.svg index fd14834d12..1dbeec5f92 100644 --- a/application/src/main/data/json/system/scada_symbols/left-bottom-elbow-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/left-bottom-elbow-connector-hp.svg @@ -3,7 +3,7 @@ "description": "Left bottom elbow connector", "widgetSizeX": 1, "widgetSizeY": 1, - "stateRenderFunction": "const {\n flowAnimation,\n animationDirection: flowDirection,\n flowAnimationSpeed\n} = ctx.values;\nconst {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\nconst line = ctx.tags.line[0].attr('d');\nconst animation = ctx.tags.animationGroup[0];\nconst offset = Date.now() % 1000;\nconst duration = 1 / flowAnimationSpeed;\n\nconst prevFlowAnimation = animation.remember('flowAnimation');\nconst prevFlowDirection = animation.remember('flowDirection');\nconst prevFlowDuration = animation.remember('flowDuration');\n\nif (flowAnimation && flowAnimation !== prevFlowAnimation) {\n animation.remember('flowAnimation', flowAnimation);\n animation.remember('flowDuration', duration);\n animation.remember('flowDirection', flowDirection);\n animateFlow(offset, flowDirection);\n} else if (flowAnimation && flowDirection !== prevFlowDirection) {\n animation.remember('flowDirection', flowDirection);\n animateFlow(offset, flowDirection);\n} else if (flowAnimation && duration !== prevFlowDuration) {\n animation.remember('flowDuration', duration);\n animation.findOne('animate').attr('dur', `${duration}s`) ;\n} else if (!flowAnimation && prevFlowAnimation) {\n animation.remember('flowAnimation', null);\n animation.clear();\n}\n\nfunction animateFlow(offset, flowDirection) {\n animation.clear();\n const dashArray = `${dashWidth}${dashGap ? ` ${dashGap}` : ''}`;\n const value = flowDirection ? `${dashWidth + (dashGap || dashWidth)}` : `-${dashWidth + (dashGap || dashWidth)}`;\n\n animation.add(``);\n}\n", + "stateRenderFunction": "const {\n flowAnimation,\n animationDirection: flowDirection,\n flowAnimationSpeed\n} = ctx.values;\nconst {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\nconst line = ctx.tags.line[0].attr('d');\nconst lineReversed = 'M 100,200 L 100,125 Q 100,100 75,100 L 0, 100';\nconst animation = ctx.tags.animationGroup[0];\nconst duration = 1 / flowAnimationSpeed;\n\nlet animateFlow = ctx.api.connectorAnimation(animation);\n\nif (flowAnimation) {\n if (!animateFlow) {\n animateFlow = ctx.api.connectorAnimate(animation, line, lineReversed).flowAppearance(lineWidth, lineColor, dashCap, dashWidth, dashGap).duration(duration).direction(flowDirection).play();\n } else {\n animateFlow.duration(duration).direction(flowDirection).play();\n }\n} else {\n if (animateFlow) {\n animateFlow.finish();\n }\n}\n", "tags": [ { "tag": "line", @@ -140,13 +140,14 @@ }, { "id": "mainLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 6, "required": true, "subLabel": "Main", "divider": true, "fieldSuffix": "px", + "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, @@ -155,21 +156,24 @@ }, { "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 2, "required": true, "subLabel": "Secondary", + "divider": true, "fieldSuffix": "px", + "disableOnProperty": "mainLine", + "condition": "return !model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, - "visible": true + "visible": false }, { "id": "lineColor", - "name": "{i18n:scada.symbol.line-color}", + "name": "{i18n:scada.symbol.main-line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -182,6 +186,7 @@ "type": "number", "default": 4, "subLabel": "Width", + "divider": true, "fieldSuffix": "px", "min": 1, "step": 1, @@ -193,11 +198,14 @@ "name": "{i18n:scada.symbol.flow}", "group": "{i18n:scada.symbol.animation}", "type": "color", - "default": "#C8DFF7" + "default": "#C8DFF7", + "disabled": false, + "visible": true }, { "id": "flowStyleDash", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, @@ -213,6 +221,7 @@ { "id": "flowStyleGap", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, @@ -248,5 +257,5 @@ } ] }]]> - + \ No newline at end of file diff --git a/application/src/main/data/json/system/scada_symbols/left-tee-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/left-tee-connector-hp.svg index af83a4abb3..82b8babcf7 100644 --- a/application/src/main/data/json/system/scada_symbols/left-tee-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/left-tee-connector-hp.svg @@ -3,7 +3,7 @@ "description": "Left tee connector", "widgetSizeX": 1, "widgetSizeY": 1, - "stateRenderFunction": "const {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\n\nconst leftLine = \"M0 100H97\";\nconst topLine = \"M100 100L100 0\";\nconst bottomLine = \"M 100,200 V 100\";\n\nprepareFlowAnimation('left', leftLine);\nprepareFlowAnimation('top', topLine);\nprepareFlowAnimation('bottom', bottomLine);\n\nfunction prepareFlowAnimation(prefix, line) {\n const flowAnimation = ctx.values[prefix + 'Flow'];\n const flowDirection = ctx.values[prefix + 'FlowDirection'];\n const flowAnimationSpeed = ctx.values[prefix + 'FlowAnimationSpeed'];\n\n const animation = ctx.tags[prefix + 'Line'][0];\n const offset = Date.now() % 1000;\n const duration = 1 / flowAnimationSpeed;\n \n const prevFlowAnimation = animation.remember('flowAnimation');\n const prevFlowDirection = animation.remember('flowDirection');\n const prevFlowDuration = animation.remember('flowDuration');\n \n if (flowAnimation && flowAnimation !== prevFlowAnimation) {\n animation.remember('flowAnimation', flowAnimation);\n animation.remember('flowDuration', duration);\n animation.remember('flowDirection', flowDirection);\n animateFlow(animation, offset, flowDirection, duration, line);\n } else if (flowAnimation && flowDirection !== prevFlowDirection) {\n animation.remember('flowDirection', flowDirection);\n animateFlow(animation, offset, flowDirection, duration, line);\n } else if (flowAnimation && duration !== prevFlowDuration) {\n animation.remember('flowDuration', duration);\n animation.findOne('animate').attr('dur', `${duration}s`) ;\n } else if (!flowAnimation && prevFlowAnimation) {\n animation.remember('flowAnimation', null);\n animation.clear();\n }\n}\n\nfunction animateFlow(group, offset, flowDirection, duration, line) {\n group.clear();\n const dashArray = `${dashWidth}${dashGap ? ` ${dashGap}` : ''}`;\n const value = flowDirection ? `${dashWidth + (dashGap || dashWidth)}` : `-${dashWidth + (dashGap || dashWidth)}`;\n\n group.add(``);\n}", + "stateRenderFunction": "const {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\n\nconst leftLine = \"M0 100H97\";\nconst leftLineReversed = \"M 97,100 H 0\";\nconst topLine = \"M100 100L100 0\";\nconst topLineReversed = \"M 100,0 V 100\";\nconst bottomLine = \"M 100,200 V 100\";\nconst bottomLineReversed = \"M 100,100 V 200\";\n\nprepareFlowAnimation('left', leftLine, leftLineReversed);\nprepareFlowAnimation('top', topLine, topLineReversed);\nprepareFlowAnimation('bottom', bottomLine, bottomLineReversed);\n\nfunction prepareFlowAnimation(prefix, line, reversedLine) {\n const flowAnimation = ctx.values[prefix + 'Flow'];\n const flowDirection = ctx.values[prefix + 'FlowDirection'];\n const flowAnimationSpeed = ctx.values[prefix + 'FlowAnimationSpeed'];\n\n const animation = ctx.tags[prefix + 'Line'][0];\n const duration = 1 / flowAnimationSpeed;\n \n let animateFlow = ctx.api.connectorAnimation(animation);\n \n if (flowAnimation) {\n if (!animateFlow) {\n animateFlow = ctx.api.connectorAnimate(animation, line, reversedLine).flowAppearance(lineWidth, lineColor, dashCap, dashWidth, dashGap).duration(duration).direction(flowDirection).play();\n } else {\n animateFlow.duration(duration).direction(flowDirection).play();\n }\n } else {\n if (animateFlow) {\n animateFlow.finish();\n }\n }\n}", "tags": [ { "tag": "line", @@ -377,13 +377,14 @@ }, { "id": "mainLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 6, "required": true, "subLabel": "Main", "divider": true, "fieldSuffix": "px", + "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, @@ -392,21 +393,23 @@ }, { "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 2, "required": true, "subLabel": "Secondary", + "divider": true, "fieldSuffix": "px", + "condition": "return !model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, - "visible": true + "visible": false }, { "id": "lineColor", - "name": "{i18n:scada.symbol.line-color}", + "name": "{i18n:scada.symbol.main-line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -419,6 +422,7 @@ "type": "number", "default": 4, "subLabel": "Width", + "divider": true, "fieldSuffix": "px", "min": 1, "step": 1, @@ -437,6 +441,7 @@ { "id": "flowStyleDash", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, @@ -452,6 +457,7 @@ { "id": "flowStyleGap", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, @@ -487,5 +493,5 @@ } ] }]]> - + \ No newline at end of file diff --git a/application/src/main/data/json/system/scada_symbols/left-top-elbow-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/left-top-elbow-connector-hp.svg index 5b6d30ab65..3c811a2372 100644 --- a/application/src/main/data/json/system/scada_symbols/left-top-elbow-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/left-top-elbow-connector-hp.svg @@ -3,7 +3,7 @@ "description": "Left top elbow connector", "widgetSizeX": 1, "widgetSizeY": 1, - "stateRenderFunction": "const {\n flowAnimation,\n animationDirection: flowDirection,\n flowAnimationSpeed\n} = ctx.values;\nconst {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\nconst line = ctx.tags.line[0].attr('d');\nconst animation = ctx.tags.animationGroup[0];\nconst offset = Date.now() % 1000;\nconst duration = 1 / flowAnimationSpeed;\n\nconst prevFlowAnimation = animation.remember('flowAnimation');\nconst prevFlowDirection = animation.remember('flowDirection');\nconst prevFlowDuration = animation.remember('flowDuration');\n\nif (flowAnimation && flowAnimation !== prevFlowAnimation) {\n animation.remember('flowAnimation', flowAnimation);\n animation.remember('flowDuration', duration);\n animation.remember('flowDirection', flowDirection);\n animateFlow(offset, flowDirection);\n} else if (flowAnimation && flowDirection !== prevFlowDirection) {\n animation.remember('flowDirection', flowDirection);\n animateFlow(offset, flowDirection);\n} else if (flowAnimation && duration !== prevFlowDuration) {\n animation.remember('flowDuration', duration);\n animation.findOne('animate').attr('dur', `${duration}s`) ;\n} else if (!flowAnimation && prevFlowAnimation) {\n animation.remember('flowAnimation', null);\n animation.clear();\n}\n\nfunction animateFlow(offset, flowDirection) {\n animation.clear();\n const dashArray = `${dashWidth}${dashGap ? ` ${dashGap}` : ''}`;\n const value = flowDirection ? `${dashWidth + (dashGap || dashWidth)}` : `-${dashWidth + (dashGap || dashWidth)}`;\n\n animation.add(``);\n}\n", + "stateRenderFunction": "const {\n flowAnimation,\n animationDirection: flowDirection,\n flowAnimationSpeed\n} = ctx.values;\nconst {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\nconst line = ctx.tags.line[0].attr('d');\nconst lineReversed = 'M 100,0 L 100,75 Q 100,100 75,100 L 0, 100';\nconst animation = ctx.tags.animationGroup[0];\nconst duration = 1 / flowAnimationSpeed;\n\nlet animateFlow = ctx.api.connectorAnimation(animation);\n\nif (flowAnimation) {\n if (!animateFlow) {\n animateFlow = ctx.api.connectorAnimate(animation, line, lineReversed).flowAppearance(lineWidth, lineColor, dashCap, dashWidth, dashGap).duration(duration).direction(flowDirection).play();\n } else {\n animateFlow.duration(duration).direction(flowDirection).play();\n }\n} else {\n if (animateFlow) {\n animateFlow.finish();\n }\n}\n", "tags": [ { "tag": "line", @@ -140,13 +140,14 @@ }, { "id": "mainLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 6, "required": true, "subLabel": "Main", "divider": true, "fieldSuffix": "px", + "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, @@ -155,21 +156,23 @@ }, { "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 2, "required": true, "subLabel": "Secondary", + "divider": true, "fieldSuffix": "px", + "condition": "return !model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, - "visible": true + "visible": false }, { "id": "lineColor", - "name": "{i18n:scada.symbol.line-color}", + "name": "{i18n:scada.symbol.main-line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -182,6 +185,7 @@ "type": "number", "default": 4, "subLabel": "Width", + "divider": true, "fieldSuffix": "px", "min": 1, "step": 1, @@ -200,6 +204,7 @@ { "id": "flowStyleDash", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, @@ -215,6 +220,7 @@ { "id": "flowStyleGap", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, @@ -250,5 +256,5 @@ } ] }]]> - + \ No newline at end of file diff --git a/application/src/main/data/json/system/scada_symbols/long-horizontal-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/long-horizontal-connector-hp.svg index 3e65bd6b04..3a3296544d 100644 --- a/application/src/main/data/json/system/scada_symbols/long-horizontal-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/long-horizontal-connector-hp.svg @@ -1,10 +1,9 @@ - - { +<svg xmlns="http://www.w3.org/2000/svg" xmlns:tb="https://thingsboard.io/svg" width="400" height="200" fill="none" version="1.1" viewBox="0 0 400 200"><tb:metadata xmlns=""><![CDATA[{ "title": "HP Long horizontal connector", "description": "Long horizontal connector with an optional directional arrow to visually indicate flow.", "widgetSizeX": 2, "widgetSizeY": 1, - "stateRenderFunction": "const {\n flowAnimation,\n arrowDirection: flowDirection,\n flowAnimationSpeed\n} = ctx.values;\nconst {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\nconst line = ctx.tags.line[0].attr('d');\nconst animation = ctx.tags.animationGroup[0];\nconst offset = Date.now() % 1000;\nconst duration = 1 / flowAnimationSpeed;\n\nconst prevFlowAnimation = animation.remember('flowAnimation');\nconst prevFlowDirection = animation.remember('flowDirection');\nconst prevFlowDuration = animation.remember('flowDuration');\n\nif (flowAnimation && flowAnimation !== prevFlowAnimation) {\n animation.remember('flowAnimation', flowAnimation);\n animation.remember('flowDuration', duration);\n animation.remember('flowDirection', flowDirection);\n animateFlow(offset, flowDirection);\n} else if (flowAnimation && flowDirection !== prevFlowDirection) {\n animation.remember('flowDirection', flowDirection);\n animateFlow(offset, flowDirection);\n} else if (flowAnimation && duration !== prevFlowDuration) {\n animation.remember('flowDuration', duration);\n animation.findOne('animate').attr('dur', `${duration}s`) ;\n} else if (!flowAnimation && prevFlowAnimation) {\n animation.remember('flowAnimation', null);\n animation.clear();\n}\n\nfunction animateFlow(offset, flowDirection) {\n animation.clear();\n const dashArray = `${dashWidth}${dashGap ? ` ${dashGap}` : ''}`;\n const value = flowDirection ? `${dashWidth + (dashGap || dashWidth)}` : `-${dashWidth + (dashGap || dashWidth)}`;\n\n animation.add(`<path style=\"stroke-dasharray: ${dashArray}; stroke-linecap: ${dashCap}; stroke-dashoffset: 0;\" d=\"${line}\" stroke-miterlimit=\"10\" fill=\"none\" stroke=\"${lineColor}\" stroke-width=\"${lineWidth}\"><animate attributeName=\"stroke-dashoffset\" values=\"${value};0\" dur=\"${duration}s\" begin=\"-${offset}ms\" calcMode=\"linear\" repeatCount=\"indefinite\" /></path>`);\n}\n", + "stateRenderFunction": "const {\n flowAnimation,\n arrowDirection: flowDirection,\n flowAnimationSpeed\n} = ctx.values;\nconst {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\nconst line = ctx.tags.line[0].attr('d');\nconst lineReversed = 'M 400,100 H 0';\nconst animation = ctx.tags.animationGroup[0];\nconst duration = 1 / flowAnimationSpeed;\n\nlet animateFlow = ctx.api.connectorAnimation(animation);\n\nif (flowAnimation) {\n if (!animateFlow) {\n animateFlow = ctx.api.connectorAnimate(animation, line, lineReversed).flowAppearance(lineWidth, lineColor, dashCap, dashWidth, dashGap).duration(duration).direction(flowDirection).play();\n } else {\n animateFlow.duration(duration).direction(flowDirection).play();\n }\n} else {\n if (animateFlow) {\n animateFlow.finish();\n }\n}\n", "tags": [ { "tag": "arrow", @@ -177,27 +176,30 @@ }, { "id": "mainLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 6, "required": true, "subLabel": "Main", "divider": true, "fieldSuffix": "px", + "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, - "visible": true + "visible": false }, { "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 2, "required": true, "subLabel": "Secondary", + "divider": true, "fieldSuffix": "px", + "condition": "return !model.mainLine;", "min": 0, "max": 99, "step": 1, @@ -206,7 +208,7 @@ }, { "id": "lineColor", - "name": "{i18n:scada.symbol.line-color}", + "name": "{i18n:scada.symbol.main-line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -219,6 +221,7 @@ "type": "number", "default": 4, "subLabel": "Width", + "divider": true, "fieldSuffix": "px", "min": 1, "step": 1, @@ -230,11 +233,14 @@ "name": "{i18n:scada.symbol.flow}", "group": "{i18n:scada.symbol.animation}", "type": "color", - "default": "#C8DFF7" + "default": "#C8DFF7", + "disabled": false, + "visible": true }, { "id": "flowStyleDash", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, @@ -250,6 +256,7 @@ { "id": "flowStyleGap", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, @@ -285,5 +292,5 @@ } ] } - + \ No newline at end of file diff --git a/application/src/main/data/json/system/scada_symbols/long-vertical-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/long-vertical-connector-hp.svg index b5c9730842..e66fd62c2e 100644 --- a/application/src/main/data/json/system/scada_symbols/long-vertical-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/long-vertical-connector-hp.svg @@ -3,7 +3,7 @@ "description": "Long vertical connector with an optional directional arrow to visually indicate flow.", "widgetSizeX": 1, "widgetSizeY": 2, - "stateRenderFunction": "const {\n flowAnimation,\n arrowDirection: flowDirection,\n flowAnimationSpeed\n} = ctx.values;\nconst {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\nconst line = ctx.tags.line[0].attr('d');\nconst animation = ctx.tags.animationGroup[0];\nconst offset = Date.now() % 1000;\nconst duration = 1 / flowAnimationSpeed;\n\nconst prevFlowAnimation = animation.remember('flowAnimation');\nconst prevFlowDirection = animation.remember('flowDirection');\nconst prevFlowDuration = animation.remember('flowDuration');\n\nif (flowAnimation && flowAnimation !== prevFlowAnimation) {\n animation.remember('flowAnimation', flowAnimation);\n animation.remember('flowDuration', duration);\n animation.remember('flowDirection', flowDirection);\n animateFlow(offset, flowDirection);\n} else if (flowAnimation && flowDirection !== prevFlowDirection) {\n animation.remember('flowDirection', flowDirection);\n animateFlow(offset, flowDirection);\n} else if (flowAnimation && duration !== prevFlowDuration) {\n animation.remember('flowDuration', duration);\n animation.findOne('animate').attr('dur', `${duration}s`) ;\n} else if (!flowAnimation && prevFlowAnimation) {\n animation.remember('flowAnimation', null);\n animation.clear();\n}\n\nfunction animateFlow(offset, flowDirection) {\n animation.clear();\n const dashArray = `${dashWidth}${dashGap ? ` ${dashGap}` : ''}`;\n const value = flowDirection ? `${dashWidth + (dashGap || dashWidth)}` : `-${dashWidth + (dashGap || dashWidth)}`;\n\n animation.add(``);\n}\n", + "stateRenderFunction": "const {\n flowAnimation,\n arrowDirection: flowDirection,\n flowAnimationSpeed\n} = ctx.values;\nconst {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\nconst line = ctx.tags.line[0].attr('d');\nconst lineReversed = 'M 100,0 V 400';\nconst animation = ctx.tags.animationGroup[0];\nconst duration = 1 / flowAnimationSpeed;\n\nlet animateFlow = ctx.api.connectorAnimation(animation);\n\nif (flowAnimation) {\n if (!animateFlow) {\n animateFlow = ctx.api.connectorAnimate(animation, line, lineReversed).flowAppearance(lineWidth, lineColor, dashCap, dashWidth, dashGap).duration(duration).direction(flowDirection).play();\n } else {\n animateFlow.duration(duration).direction(flowDirection).play();\n }\n} else {\n if (animateFlow) {\n animateFlow.finish();\n }\n}\n\n", "tags": [ { "tag": "arrow", @@ -176,13 +176,14 @@ }, { "id": "mainLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 6, "required": true, "subLabel": "Main", "divider": true, "fieldSuffix": "px", + "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, @@ -191,21 +192,23 @@ }, { "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 2, "required": true, "subLabel": "Secondary", + "divider": true, "fieldSuffix": "px", + "condition": "return !model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, - "visible": true + "visible": false }, { "id": "lineColor", - "name": "{i18n:scada.symbol.line-color}", + "name": "{i18n:scada.symbol.main-line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -218,6 +221,7 @@ "type": "number", "default": 4, "subLabel": "Width", + "divider": true, "fieldSuffix": "px", "min": 1, "step": 1, @@ -229,11 +233,14 @@ "name": "{i18n:scada.symbol.flow}", "group": "{i18n:scada.symbol.animation}", "type": "color", - "default": "#C8DFF7" + "default": "#C8DFF7", + "disabled": false, + "visible": true }, { "id": "flowStyleDash", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, @@ -249,6 +256,7 @@ { "id": "flowStyleGap", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, @@ -284,5 +292,5 @@ } ] }]]> - + \ No newline at end of file diff --git a/application/src/main/data/json/system/scada_symbols/right-tee-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/right-tee-connector-hp.svg index 62aecb065d..e40936c152 100644 --- a/application/src/main/data/json/system/scada_symbols/right-tee-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/right-tee-connector-hp.svg @@ -3,7 +3,7 @@ "description": "Right tee connector", "widgetSizeX": 1, "widgetSizeY": 1, - "stateRenderFunction": "const {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\n\nconst topLine = \"M100 100L100 0\";\nconst rightLine = \"M103 100H200\";\nconst bottomLine = \"M 100,200 V 100\";\n\nprepareFlowAnimation('top', topLine);\nprepareFlowAnimation('right', rightLine);\nprepareFlowAnimation('bottom', bottomLine);\n\nfunction prepareFlowAnimation(prefix, line) {\n const flowAnimation = ctx.values[prefix + 'Flow'];\n const flowDirection = ctx.values[prefix + 'FlowDirection'];\n const flowAnimationSpeed = ctx.values[prefix + 'FlowAnimationSpeed'];\n\n const animation = ctx.tags[prefix + 'Line'][0];\n const offset = Date.now() % 1000;\n const duration = 1 / flowAnimationSpeed;\n \n const prevFlowAnimation = animation.remember('flowAnimation');\n const prevFlowDirection = animation.remember('flowDirection');\n const prevFlowDuration = animation.remember('flowDuration');\n \n if (flowAnimation && flowAnimation !== prevFlowAnimation) {\n animation.remember('flowAnimation', flowAnimation);\n animation.remember('flowDuration', duration);\n animation.remember('flowDirection', flowDirection);\n animateFlow(animation, offset, flowDirection, duration, line);\n } else if (flowAnimation && flowDirection !== prevFlowDirection) {\n animation.remember('flowDirection', flowDirection);\n animateFlow(animation, offset, flowDirection, duration, line);\n } else if (flowAnimation && duration !== prevFlowDuration) {\n animation.remember('flowDuration', duration);\n animation.findOne('animate').attr('dur', `${duration}s`) ;\n } else if (!flowAnimation && prevFlowAnimation) {\n animation.remember('flowAnimation', null);\n animation.clear();\n }\n}\n\nfunction animateFlow(group, offset, flowDirection, duration, line) {\n group.clear();\n const dashArray = `${dashWidth}${dashGap ? ` ${dashGap}` : ''}`;\n const value = flowDirection ? `${dashWidth + (dashGap || dashWidth)}` : `-${dashWidth + (dashGap || dashWidth)}`;\n\n group.add(``);\n}", + "stateRenderFunction": "const {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\n\nconst topLine = \"M100 100L100 0\";\nconst topLineReversed = \"M 100,0 V 100\";\nconst rightLine = \"M103 100H200\";\nconst rightLineReversed = \"M 200,100 H 103\";\nconst bottomLine = \"M 100,200 V 100\";\nconst bottomLineReversed = \"M 100,100 V 200\";\n\nprepareFlowAnimation('top', topLine, topLineReversed);\nprepareFlowAnimation('right', rightLine, rightLineReversed);\nprepareFlowAnimation('bottom', bottomLine, bottomLineReversed);\n\nfunction prepareFlowAnimation(prefix, line, reversedLine) {\n const flowAnimation = ctx.values[prefix + 'Flow'];\n const flowDirection = ctx.values[prefix + 'FlowDirection'];\n const flowAnimationSpeed = ctx.values[prefix + 'FlowAnimationSpeed'];\n\n const animation = ctx.tags[prefix + 'Line'][0];\n const duration = 1 / flowAnimationSpeed;\n \n let animateFlow = ctx.api.connectorAnimation(animation);\n \n if (flowAnimation) {\n if (!animateFlow) {\n animateFlow = ctx.api.connectorAnimate(animation, line, reversedLine).flowAppearance(lineWidth, lineColor, dashCap, dashWidth, dashGap).duration(duration).direction(flowDirection).play();\n } else {\n animateFlow.duration(duration).direction(flowDirection).play();\n }\n } else {\n if (animateFlow) {\n animateFlow.finish();\n }\n }\n}", "tags": [ { "tag": "line", @@ -377,13 +377,14 @@ }, { "id": "mainLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 6, "required": true, "subLabel": "Main", "divider": true, "fieldSuffix": "px", + "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, @@ -392,21 +393,23 @@ }, { "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 2, "required": true, "subLabel": "Secondary", + "divider": true, "fieldSuffix": "px", + "condition": "return !model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, - "visible": true + "visible": false }, { "id": "lineColor", - "name": "{i18n:scada.symbol.line-color}", + "name": "{i18n:scada.symbol.main-line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -419,6 +422,7 @@ "type": "number", "default": 4, "subLabel": "Width", + "divider": true, "fieldSuffix": "px", "min": 1, "step": 1, @@ -437,6 +441,7 @@ { "id": "flowStyleDash", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, @@ -452,6 +457,7 @@ { "id": "flowStyleGap", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, diff --git a/application/src/main/data/json/system/scada_symbols/top-right-elbow-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/top-right-elbow-connector-hp.svg index 8ec4e3cc65..8c5224c9f8 100644 --- a/application/src/main/data/json/system/scada_symbols/top-right-elbow-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/top-right-elbow-connector-hp.svg @@ -3,7 +3,7 @@ "description": "Top right elbow connector", "widgetSizeX": 1, "widgetSizeY": 1, - "stateRenderFunction": "const {\n flowAnimation,\n animationDirection: flowDirection,\n flowAnimationSpeed\n} = ctx.values;\nconst {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\nconst line = ctx.tags.line[0].attr('d');\nconst animation = ctx.tags.animationGroup[0];\nconst offset = Date.now() % 1000;\nconst duration = 1 / flowAnimationSpeed;\n\nconst prevFlowAnimation = animation.remember('flowAnimation');\nconst prevFlowDirection = animation.remember('flowDirection');\nconst prevFlowDuration = animation.remember('flowDuration');\n\nif (flowAnimation && flowAnimation !== prevFlowAnimation) {\n animation.remember('flowAnimation', flowAnimation);\n animation.remember('flowDuration', duration);\n animation.remember('flowDirection', flowDirection);\n animateFlow(offset, flowDirection);\n} else if (flowAnimation && flowDirection !== prevFlowDirection) {\n animation.remember('flowDirection', flowDirection);\n animateFlow(offset, flowDirection);\n} else if (flowAnimation && duration !== prevFlowDuration) {\n animation.remember('flowDuration', duration);\n animation.findOne('animate').attr('dur', `${duration}s`) ;\n} else if (!flowAnimation && prevFlowAnimation) {\n animation.remember('flowAnimation', null);\n animation.clear();\n}\n\nfunction animateFlow(offset, flowDirection) {\n animation.clear();\n const dashArray = `${dashWidth}${dashGap ? ` ${dashGap}` : ''}`;\n const value = flowDirection ? `-${dashWidth + (dashGap || dashWidth)}` : `${dashWidth + (dashGap || dashWidth)}`;\n\n animation.add(``);\n}\n", + "stateRenderFunction": "const {\n flowAnimation,\n animationDirection: flowDirection,\n flowAnimationSpeed\n} = ctx.values;\nconst {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\nconst line = ctx.tags.line[0].attr('d');\nconst lineReversed = 'M 200,100 L 125,100 Q 100,100 100,75 L 100, 0';\nconst animation = ctx.tags.animationGroup[0];\nconst duration = 1 / flowAnimationSpeed;\n\nlet animateFlow = ctx.api.connectorAnimation(animation);\n\nif (flowAnimation) {\n if (!animateFlow) {\n animateFlow = ctx.api.connectorAnimate(animation, line, lineReversed).flowAppearance(lineWidth, lineColor, dashCap, dashWidth, dashGap).duration(duration).direction(flowDirection).play();\n } else {\n animateFlow.duration(duration).direction(flowDirection).play();\n }\n} else {\n if (animateFlow) {\n animateFlow.finish();\n }\n}\n", "tags": [ { "tag": "line", @@ -140,13 +140,14 @@ }, { "id": "mainLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 6, "required": true, "subLabel": "Main", "divider": true, "fieldSuffix": "px", + "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, @@ -155,21 +156,23 @@ }, { "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 2, "required": true, "subLabel": "Secondary", + "divider": true, "fieldSuffix": "px", + "condition": "return !model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, - "visible": true + "visible": false }, { "id": "lineColor", - "name": "{i18n:scada.symbol.line-color}", + "name": "{i18n:scada.symbol.main-line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -182,6 +185,7 @@ "type": "number", "default": 4, "subLabel": "Width", + "divider": true, "fieldSuffix": "px", "min": 1, "step": 1, @@ -200,6 +204,7 @@ { "id": "flowStyleDash", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, @@ -215,6 +220,7 @@ { "id": "flowStyleGap", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, @@ -250,5 +256,5 @@ } ] }]]> - + \ No newline at end of file diff --git a/application/src/main/data/json/system/scada_symbols/top-tee-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/top-tee-connector-hp.svg index e4561a8347..101799fd4b 100644 --- a/application/src/main/data/json/system/scada_symbols/top-tee-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/top-tee-connector-hp.svg @@ -3,7 +3,7 @@ "description": "Top tee connector", "widgetSizeX": 1, "widgetSizeY": 1, - "stateRenderFunction": "const {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\n\nconst leftLine = \"M0 100H100\";\nconst topLine = \"M100 97L100 0\";\nconst rightLine = \"M100 100H200\";\n\nprepareFlowAnimation('left', leftLine);\nprepareFlowAnimation('top', topLine);\nprepareFlowAnimation('right', rightLine);\n\nfunction prepareFlowAnimation(prefix, line) {\n const flowAnimation = ctx.values[prefix + 'Flow'];\n const flowDirection = ctx.values[prefix + 'FlowDirection'];\n const flowAnimationSpeed = ctx.values[prefix + 'FlowAnimationSpeed'];\n\n const animation = ctx.tags[prefix + 'Line'][0];\n const offset = Date.now() % 1000;\n const duration = 1 / flowAnimationSpeed;\n \n const prevFlowAnimation = animation.remember('flowAnimation');\n const prevFlowDirection = animation.remember('flowDirection');\n const prevFlowDuration = animation.remember('flowDuration');\n \n if (flowAnimation && flowAnimation !== prevFlowAnimation) {\n animation.remember('flowAnimation', flowAnimation);\n animation.remember('flowDuration', duration);\n animation.remember('flowDirection', flowDirection);\n animateFlow(animation, offset, flowDirection, duration, line);\n } else if (flowAnimation && flowDirection !== prevFlowDirection) {\n animation.remember('flowDirection', flowDirection);\n animateFlow(animation, offset, flowDirection, duration, line);\n } else if (flowAnimation && duration !== prevFlowDuration) {\n animation.remember('flowDuration', duration);\n animation.findOne('animate').attr('dur', `${duration}s`) ;\n } else if (!flowAnimation && prevFlowAnimation) {\n animation.remember('flowAnimation', null);\n animation.clear();\n }\n}\n\nfunction animateFlow(group, offset, flowDirection, duration, line) {\n group.clear();\n const dashArray = `${dashWidth}${dashGap ? ` ${dashGap}` : ''}`;\n const value = flowDirection ? `${dashWidth + (dashGap || dashWidth)}` : `-${dashWidth + (dashGap || dashWidth)}`;\n\n group.add(``);\n}", + "stateRenderFunction": "const {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\n\nconst leftLine = \"M0 100H100\";\nconst leftLineReversed = \"M 100,100 H 0\";\nconst topLine = \"M100 97L100 0\";\nconst topLineReversed = \"M 100,0 V 97\";\nconst rightLine = \"M100 100H200\";\nconst rightLineReversed = \"M 200,100 H 100\";\n\nprepareFlowAnimation('left', leftLine, leftLineReversed);\nprepareFlowAnimation('top', topLine, topLineReversed);\nprepareFlowAnimation('right', rightLine, rightLineReversed);\n\nfunction prepareFlowAnimation(prefix, line, reversedLine) {\n const flowAnimation = ctx.values[prefix + 'Flow'];\n const flowDirection = ctx.values[prefix + 'FlowDirection'];\n const flowAnimationSpeed = ctx.values[prefix + 'FlowAnimationSpeed'];\n\n const animation = ctx.tags[prefix + 'Line'][0];\n const duration = 1 / flowAnimationSpeed;\n \n let animateFlow = ctx.api.connectorAnimation(animation);\n \n if (flowAnimation) {\n if (!animateFlow) {\n animateFlow = ctx.api.connectorAnimate(animation, line, reversedLine).flowAppearance(lineWidth, lineColor, dashCap, dashWidth, dashGap).duration(duration).direction(flowDirection).play();\n } else {\n animateFlow.duration(duration).direction(flowDirection).play();\n }\n } else {\n if (animateFlow) {\n animateFlow.finish();\n }\n }\n}", "tags": [ { "tag": "line", @@ -377,13 +377,14 @@ }, { "id": "mainLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 6, "required": true, "subLabel": "Main", "divider": true, "fieldSuffix": "px", + "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, @@ -392,21 +393,23 @@ }, { "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 2, "required": true, "subLabel": "Secondary", + "divider": true, "fieldSuffix": "px", + "condition": "return !model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, - "visible": true + "visible": false }, { "id": "lineColor", - "name": "{i18n:scada.symbol.line-color}", + "name": "{i18n:scada.symbol.main-line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -419,6 +422,7 @@ "type": "number", "default": 4, "subLabel": "Width", + "divider": true, "fieldSuffix": "px", "min": 1, "step": 1, @@ -437,6 +441,7 @@ { "id": "flowStyleDash", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, @@ -452,6 +457,7 @@ { "id": "flowStyleGap", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, diff --git a/application/src/main/data/json/system/scada_symbols/vertical-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/vertical-connector-hp.svg index cfaf6793ea..698a910f8d 100644 --- a/application/src/main/data/json/system/scada_symbols/vertical-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/vertical-connector-hp.svg @@ -3,7 +3,7 @@ "description": "Vertical connector with an optional directional arrow to visually indicate flow.", "widgetSizeX": 1, "widgetSizeY": 1, - "stateRenderFunction": "const {\n flowAnimation,\n arrowDirection: flowDirection,\n flowAnimationSpeed\n} = ctx.values;\nconst {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\nconst line = ctx.tags.line[0].attr('d');\nconst animation = ctx.tags.animationGroup[0];\nconst offset = Date.now() % 1000;\nconst duration = 1 / flowAnimationSpeed;\n\nconst prevFlowAnimation = animation.remember('flowAnimation');\nconst prevFlowDirection = animation.remember('flowDirection');\nconst prevFlowDuration = animation.remember('flowDuration');\n\nif (flowAnimation && flowAnimation !== prevFlowAnimation) {\n animation.remember('flowAnimation', flowAnimation);\n animation.remember('flowDuration', duration);\n animation.remember('flowDirection', flowDirection);\n animateFlow(offset, flowDirection);\n} else if (flowAnimation && flowDirection !== prevFlowDirection) {\n animation.remember('flowDirection', flowDirection);\n animateFlow(offset, flowDirection);\n} else if (flowAnimation && duration !== prevFlowDuration) {\n animation.remember('flowDuration', duration);\n animation.findOne('animate').attr('dur', `${duration}s`) ;\n} else if (!flowAnimation && prevFlowAnimation) {\n animation.remember('flowAnimation', null);\n animation.clear();\n}\n\nfunction animateFlow(offset, flowDirection) {\n animation.clear();\n const dashArray = `${dashWidth}${dashGap ? ` ${dashGap}` : ''}`;\n const value = flowDirection ? `${dashWidth + (dashGap || dashWidth)}` : `-${dashWidth + (dashGap || dashWidth)}`;\n\n animation.add(``);\n}\n", + "stateRenderFunction": "const {\n flowAnimation,\n arrowDirection: flowDirection,\n flowAnimationSpeed\n} = ctx.values;\nconst {\n flowAnimationWidth: lineWidth,\n flowAnimationColor: lineColor,\n flowStyleDash: dashWidth,\n flowStyleGap: dashGap,\n flowDashCap: dashCap\n} = ctx.properties;\nconst line = ctx.tags.line[0].attr('d');\nconst lineReversed = 'M 100,0 V 200';\nconst animation = ctx.tags.animationGroup[0];\nconst duration = 1 / flowAnimationSpeed;\n\nlet animateFlow = ctx.api.connectorAnimation(animation);\n\nif (flowAnimation) {\n if (!animateFlow) {\n animateFlow = ctx.api.connectorAnimate(animation, line, lineReversed).flowAppearance(lineWidth, lineColor, dashCap, dashWidth, dashGap).duration(duration).direction(flowDirection).play();\n } else {\n animateFlow.duration(duration).direction(flowDirection).play();\n }\n} else {\n if (animateFlow) {\n animateFlow.finish();\n }\n}\n", "tags": [ { "tag": "arrow", @@ -176,13 +176,14 @@ }, { "id": "mainLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 6, "required": true, "subLabel": "Main", "divider": true, "fieldSuffix": "px", + "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, @@ -191,21 +192,23 @@ }, { "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.line}", + "name": "{i18n:scada.symbol.main-line}", "type": "number", "default": 2, "required": true, "subLabel": "Secondary", + "divider": true, "fieldSuffix": "px", + "condition": "return !model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, - "visible": true + "visible": false }, { "id": "lineColor", - "name": "{i18n:scada.symbol.line-color}", + "name": "{i18n:scada.symbol.main-line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -218,6 +221,7 @@ "type": "number", "default": 4, "subLabel": "Width", + "divider": true, "fieldSuffix": "px", "min": 1, "step": 1, @@ -229,11 +233,14 @@ "name": "{i18n:scada.symbol.flow}", "group": "{i18n:scada.symbol.animation}", "type": "color", - "default": "#C8DFF7" + "default": "#C8DFF7", + "disabled": false, + "visible": true }, { "id": "flowStyleDash", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, @@ -249,6 +256,7 @@ { "id": "flowStyleGap", "name": "{i18n:scada.symbol.flow-style}", + "hint": "{i18n:scada.symbol.flow-style-hint}", "group": "{i18n:scada.symbol.animation}", "type": "number", "default": 10, @@ -284,5 +292,5 @@ } ] }]]> - + \ No newline at end of file diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/scada/scada-symbol.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/scada/scada-symbol.models.ts index abe570fa3c..ee63dc59a6 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/scada/scada-symbol.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/scada/scada-symbol.models.ts @@ -83,6 +83,10 @@ export interface ScadaSymbolApi { cssAnimation: (element: Element) => ScadaSymbolAnimation | undefined; resetCssAnimation: (element: Element) => void; finishCssAnimation: (element: Element) => void; + connectorAnimation:(element: Element) => ConnectorScadaSymbolAnimation | undefined; + connectorAnimate:(element: Element) => ConnectorScadaSymbolAnimation; + resetConnectorAnimation: (element: Element) => void; + finishConnectorAnimation: (element: Element) => void; disable: (element: Element | Element[]) => void; enable: (element: Element | Element[]) => void; callAction: (event: Event, behaviorId: string, value?: any, observer?: Partial>) => void; @@ -186,6 +190,8 @@ const tbNamespaceRegex = //gm const tbTagRegex = /tb:tag="([^"]*)"/gms; +let syncTime = Date.now(); + const generateElementId = () => { const id = guid(); const firstChar = id.charAt(0); @@ -485,6 +491,7 @@ export class ScadaSymbolObject { private settings: ScadaSymbolObjectSettings; private context: ScadaSymbolContext; private cssAnimations: CssScadaSymbolAnimations; + private connectorAnimations: ScadaSymbolFlowConnectorAnimations; private svgShape: Svg; private box: Box; @@ -604,6 +611,7 @@ export class ScadaSymbolObject { private init() { this.cssAnimations = new CssScadaSymbolAnimations(this.svgShape, this.raf); + this.connectorAnimations = new ScadaSymbolFlowConnectorAnimations(); this.context = { api: { generateElementId: () => generateElementId(), @@ -615,6 +623,10 @@ export class ScadaSymbolObject { cssAnimation: this.cssAnimation.bind(this), resetCssAnimation: this.resetCssAnimation.bind(this), finishCssAnimation: this.finishCssAnimation.bind(this), + connectorAnimation: this.connectorAnimation.bind(this), + connectorAnimate: this.connectorAnimate.bind(this), + resetConnectorAnimation: this.resetConnectorAnimation.bind(this), + finishConnectorAnimation: this.finishConnectorAnimation.bind(this), disable: this.disableElement.bind(this), enable: this.enableElement.bind(this), callAction: this.callAction.bind(this), @@ -959,6 +971,22 @@ export class ScadaSymbolObject { this.cssAnimations.finishAnimation(element); } + private connectorAnimate(element: Element, path: string, reversedPath: string): ConnectorScadaSymbolAnimation { + return this.connectorAnimations.animate(element, path, reversedPath); + } + + private connectorAnimation(element: Element): ConnectorScadaSymbolAnimation | undefined { + return this.connectorAnimations.animation(element); + } + + private resetConnectorAnimation(element: Element) { + this.connectorAnimations.resetAnimation(element); + } + + private finishConnectorAnimation(element: Element) { + this.connectorAnimations.finishAnimation(element); + } + private disableElement(e: Element | Element[]) { this.elements(e).forEach(element => { element.attr({'pointer-events': 'none'}); @@ -1108,6 +1136,20 @@ interface ScadaSymbolAnimation { } +const scadaSymbolConnectorFlowAnimationId = 'scadaSymbolConnectorFlowAnimation'; + +type StrokeLineCap = 'butt' | 'round '| 'square'; + +interface ConnectorScadaSymbolAnimation { + play(): void; + stop(): void; + finish(): void; + + flowAppearance(width: number, color: string, lineCap: StrokeLineCap, dashWidth: number, dashGap: number): ConnectorScadaSymbolAnimation; + duration(speed: number): ConnectorScadaSymbolAnimation; + direction(direction: boolean): ConnectorScadaSymbolAnimation; +} + class CssScadaSymbolAnimations { constructor(private svgShape: Svg, private raf: RafService) {} @@ -1159,6 +1201,135 @@ class CssScadaSymbolAnimations { } } +class ScadaSymbolFlowConnectorAnimations { + constructor() {} + + public animate(element: Element, path = '', reversedPath = ''): ConnectorScadaSymbolAnimation { + this.checkOldAnimation(element); + return this.setupAnimation(element, this.createAnimation(element, path, reversedPath)); + } + + public animation(element: Element): ConnectorScadaSymbolAnimation | undefined { + return element.remember(scadaSymbolConnectorFlowAnimationId); + } + + public resetAnimation(element: Element) { + const animation: ConnectorScadaSymbolAnimation = element.remember(scadaSymbolConnectorFlowAnimationId); + if (animation) { + animation.stop(); + element.remember(scadaSymbolConnectorFlowAnimationId, null); + } + } + + public finishAnimation(element: Element) { + const animation: ConnectorScadaSymbolAnimation = element.remember(scadaSymbolConnectorFlowAnimationId); + if (animation) { + animation.finish(); + element.remember(scadaSymbolConnectorFlowAnimationId, null); + } + } + + private setupAnimation(element: Element, animation: ConnectorScadaSymbolAnimation): ConnectorScadaSymbolAnimation { + element.remember(scadaSymbolConnectorFlowAnimationId, animation); + return animation; + } + + private checkOldAnimation(element: Element) { + const previousAnimation: ConnectorScadaSymbolAnimation = element.remember(scadaSymbolConnectorFlowAnimationId); + if (previousAnimation) { + previousAnimation.finish(); + } + } + + private createAnimation(element: Element, path: string, reversedPath: string): ConnectorScadaSymbolAnimation { + return new FlowConnectorAnimation(element, path, reversedPath); + } +} + +class FlowConnectorAnimation implements ConnectorScadaSymbolAnimation { + + private readonly _path: string; + private readonly _reversedPath: string; + private readonly _animation: Element; + + private _duration: number = 1; + private _lineColor: string = '#C8DFF7'; + private _lineWidth: number = 4; + private _strokeLineCap: StrokeLineCap = 'butt'; + private _dashWidth: number = 10; + private _dashGap: number = 10; + private _direction: boolean = true; + + constructor(private element: Element, + path: string, + pathReversed: string) { + this._path = path; + this._reversedPath = pathReversed; + + const dashArray = `${this._dashWidth} ${this._dashGap}`; + const values = `${this._dashWidth + this._dashGap};0`; + + this._animation = SVG( + `` + + `` + ); + } + + public play() { + if (!this.element.node.childElementCount) { + this.element.add(this._animation); + } + if (!syncTime) { + syncTime = Date.now(); + } + const animateElement = this.element.node.getElementsByTagName('animate')[0]; + const offset = ((Date.now() - syncTime) % 1000) * -1; + (animateElement as SVGAnimationElement).beginElementAt(offset); + } + + public stop() { + const animateElement = this.element.node.getElementsByTagName('animate')[0]; + (animateElement as SVGAnimationElement)?.endElement(); + } + + public finish() { + this.element.findOne('path')?.remove(); + } + + public flowAppearance(width: number, color: string, linecap: StrokeLineCap, dashWidth: number, dashGap: number): this { + const totalLength = (this._animation.node as SVGPathElement).getTotalLength(); + let offset = 0; + if ((totalLength % 100) !== 0) { + const clientWidth = totalLength < 100 ? 100 : this.element.node.ownerSVGElement.clientWidth; + const clientWidthDash = clientWidth / (dashWidth + dashGap); + const totalLengthDash = totalLength / clientWidthDash; + offset = ((dashWidth + dashGap) - totalLengthDash) / 2; + } + this._lineColor = color; + this._lineWidth = width; + this._strokeLineCap = linecap; + this._dashWidth = dashWidth - offset; + this._dashGap = dashGap - offset; + const dashArray = `${this._dashWidth}${this._dashGap ? ` ${this._dashGap}` : ''}`; + const values = `${this._dashWidth + (this._dashGap || this._dashWidth)};0`; + this._animation.stroke({width, color, linecap, dasharray: dashArray}); + this._animation.findOne('animate').attr('values', values); + return this; + } + + public duration(speed: number): this { + this._duration = speed; + this._animation.findOne('animate').attr('dur', `${speed}s`); + return this; + } + + public direction(direction: boolean): this { + this._direction = direction; + this._animation.attr('d', direction ? this._path : this._reversedPath); + return this; + } +} + interface ScadaSymbolAnimationKeyframe { stop: string; style: any; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/dynamic-form/dynamic-form-property-panel.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/dynamic-form/dynamic-form-property-panel.component.html index b32c7b5ab1..af59457abe 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/dynamic-form/dynamic-form-property-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/dynamic-form/dynamic-form-property-panel.component.html @@ -30,6 +30,12 @@ +
+
scada.behavior.hint
+ + + +
dynamic-form.property.group-title
diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/dynamic-form/dynamic-form-property-panel.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/dynamic-form/dynamic-form-property-panel.component.ts index 2df62e1259..00a614636d 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/dynamic-form/dynamic-form-property-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/dynamic-form/dynamic-form-property-panel.component.ts @@ -104,6 +104,7 @@ export class DynamicFormPropertyPanelComponent implements OnInit { { id: [this.property.id, [Validators.required]], name: [this.property.name, [Validators.required]], + hint: [this.property.hint, []], group: [this.property.group, []], type: [this.property.type, [Validators.required]], arrayItemType: [this.property.arrayItemType, [Validators.required]], diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/dynamic-form/dynamic-form.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/dynamic-form/dynamic-form.component.html index a64c178eb6..449af2b240 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/dynamic-form/dynamic-form.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/dynamic-form/dynamic-form.component.html @@ -91,9 +91,15 @@
- {{ propertyRow.label | customTranslate }} +
+ {{ propertyRow.label | customTranslate }} +
-
{{ propertyRow.label | customTranslate }}
+
+
+ {{ propertyRow.label | customTranslate }} +
+
diff --git a/ui-ngx/src/app/shared/models/dynamic-form.models.ts b/ui-ngx/src/app/shared/models/dynamic-form.models.ts index 4132eeff94..b1cc83dbd7 100644 --- a/ui-ngx/src/app/shared/models/dynamic-form.models.ts +++ b/ui-ngx/src/app/shared/models/dynamic-form.models.ts @@ -87,6 +87,7 @@ export type PropertyConditionFunction = (property: FormProperty, model: any) => export interface FormPropertyBase { id: string; name: string; + hint?: string; group?: string; type: FormPropertyType; default: any; @@ -237,6 +238,7 @@ export interface FormPropertyContainerBase { } export interface FormPropertyRow extends FormPropertyContainerBase { + hint?: string; properties?: FormProperty[]; switch?: FormProperty; rowClass?: string; @@ -362,6 +364,7 @@ const toPropertyContainers = (properties: FormProperty[], if (!propertyRow) { propertyRow = { label: property.name, + hint: property.hint, type: FormPropertyContainerType.row, properties: [], rowClass: property.rowClass, diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 72afcfc0b9..a3ba654258 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3446,6 +3446,7 @@ "flow-animation-hint": "Indicates whether animation is present in connector.", "flow": "Flow", "flow-style": "Flow style", + "flow-style-hint": "Set the Dash and Gap values so that their sum is divisible by 100 without a remainder for perfect animation synchronization.", "flow-dash-cap": "Flow dash cap", "dash-cap-butt": "Butt", "dash-cap-round": "Round", From 166475a5246fd669726e9b306e9a9885d21f16eb Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Thu, 1 May 2025 16:21:14 +0300 Subject: [PATCH 15/79] UI: Refactoring connector settings --- .../bottom-right-elbow-connector-hp.svg | 65 +++------- .../scada_symbols/bottom-tee-connector-hp.svg | 73 ++++------- .../scada_symbols/cross-connector-hp.svg | 74 ++++------- .../scada_symbols/horizontal-connector-hp.svg | 111 +++++++---------- .../left-bottom-elbow-connector-hp.svg | 74 ++++------- .../scada_symbols/left-tee-connector-hp.svg | 73 ++++------- .../left-top-elbow-connector-hp.svg | 65 +++------- .../long-horizontal-connector-hp.svg | 113 +++++++---------- .../long-vertical-connector-hp.svg | 117 +++++++----------- .../scada_symbols/right-tee-connector-hp.svg | 73 ++++------- .../top-right-elbow-connector-hp.svg | 65 +++------- .../scada_symbols/top-tee-connector-hp.svg | 73 ++++------- .../scada_symbols/vertical-connector-hp.svg | 113 +++++++---------- .../widget/lib/scada/scada-symbol.models.ts | 7 +- .../assets/locale/locale.constant-en_US.json | 13 +- 15 files changed, 376 insertions(+), 733 deletions(-) diff --git a/application/src/main/data/json/system/scada_symbols/bottom-right-elbow-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/bottom-right-elbow-connector-hp.svg index b76f7399cc..f7ffd5f167 100644 --- a/application/src/main/data/json/system/scada_symbols/bottom-right-elbow-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/bottom-right-elbow-connector-hp.svg @@ -7,7 +7,7 @@ "tags": [ { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nif (ctx.properties.mainLine) {\n element.attr({'stroke-width': ctx.properties.mainLineSize});\n} else {\n element.attr({'stroke-width': ctx.properties.secondaryLineSize});\n}", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", "actions": null } ], @@ -53,8 +53,8 @@ }, { "id": "animationDirection", - "name": "{i18n:scada.symbol.animation-direction}", - "hint": "{i18n:scada.symbol.animation-direction-hint}", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": null, "type": "value", "valueType": "BOOLEAN", @@ -131,48 +131,22 @@ ], "properties": [ { - "id": "mainLine", - "name": "{i18n:scada.symbol.main-line}", - "type": "switch", - "default": true, - "disabled": false, - "visible": true - }, - { - "id": "mainLineSize", - "name": "{i18n:scada.symbol.main-line}", + "id": "lineSize", + "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, "required": true, - "subLabel": "Main", - "divider": true, + "divider": false, "fieldSuffix": "px", - "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, "visible": true }, - { - "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.main-line}", - "type": "number", - "default": 2, - "required": true, - "subLabel": "Secondary", - "divider": true, - "fieldSuffix": "px", - "condition": "return !model.mainLine;", - "min": 0, - "max": 99, - "step": 1, - "disabled": false, - "visible": false - }, { "id": "lineColor", - "name": "{i18n:scada.symbol.main-line}", + "name": "{i18n:scada.symbol.line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -180,12 +154,11 @@ }, { "id": "flowAnimationWidth", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 4, - "subLabel": "Width", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 1, "step": 1, @@ -194,8 +167,8 @@ }, { "id": "flowAnimationColor", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "color", "default": "#C8DFF7", "disabled": false, @@ -203,14 +176,14 @@ }, { "id": "flowStyleDash", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "required": true, "subLabel": "{i18n:scada.symbol.dash}", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 0, "step": 1, @@ -219,14 +192,14 @@ }, { "id": "flowStyleGap", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "subLabel": "{i18n:scada.symbol.gap}", "fieldSuffix": "px", - "min": 1, + "min": 0, "step": 1, "disabled": false, "visible": true @@ -234,7 +207,7 @@ { "id": "flowDashCap", "name": "{i18n:scada.symbol.flow-dash-cap}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "select", "default": "butt", "items": [ diff --git a/application/src/main/data/json/system/scada_symbols/bottom-tee-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/bottom-tee-connector-hp.svg index 28e0788d35..bdbbdca81f 100644 --- a/application/src/main/data/json/system/scada_symbols/bottom-tee-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/bottom-tee-connector-hp.svg @@ -7,7 +7,7 @@ "tags": [ { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nif (ctx.properties.mainLine) {\n element.attr({'stroke-width': ctx.properties.mainLineSize});\n} else {\n element.attr({'stroke-width': ctx.properties.secondaryLineSize});\n}", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", "actions": null }, { @@ -58,8 +58,8 @@ }, { "id": "leftFlowDirection", - "name": "{i18n:scada.symbol.animation-direction}", - "hint": "{i18n:scada.symbol.animation-direction-hint}", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": "{i18n:scada.symbol.left-connector}", "type": "value", "valueType": "BOOLEAN", @@ -174,8 +174,8 @@ }, { "id": "rightFlowDirection", - "name": "{i18n:scada.symbol.flow-direction}", - "hint": "{i18n:scada.symbol.flow-direction-hint}", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": "{i18n:scada.symbol.right-connector}", "type": "value", "valueType": "BOOLEAN", @@ -290,8 +290,8 @@ }, { "id": "bottomFlowDirection", - "name": "{i18n:scada.symbol.animation-direction}", - "hint": "{i18n:scada.symbol.animation-direction-hint}", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": "{i18n:scada.symbol.bottom-connector}", "type": "value", "valueType": "BOOLEAN", @@ -368,48 +368,22 @@ ], "properties": [ { - "id": "mainLine", - "name": "{i18n:scada.symbol.main-line}", - "type": "switch", - "default": true, - "disabled": false, - "visible": true - }, - { - "id": "mainLineSize", - "name": "{i18n:scada.symbol.main-line}", + "id": "lineSize", + "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, "required": true, - "subLabel": "Main", - "divider": true, + "divider": false, "fieldSuffix": "px", - "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, "visible": true }, - { - "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.main-line}", - "type": "number", - "default": 2, - "required": true, - "subLabel": "Secondary", - "divider": true, - "fieldSuffix": "px", - "condition": "return !model.mainLine;", - "min": 0, - "max": 99, - "step": 1, - "disabled": false, - "visible": false - }, { "id": "lineColor", - "name": "{i18n:scada.symbol.main-line}", + "name": "{i18n:scada.symbol.line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -417,12 +391,11 @@ }, { "id": "flowAnimationWidth", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 4, - "subLabel": "Width", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 1, "step": 1, @@ -431,8 +404,8 @@ }, { "id": "flowAnimationColor", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "color", "default": "#C8DFF7", "disabled": false, @@ -440,14 +413,14 @@ }, { "id": "flowStyleDash", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "required": true, "subLabel": "{i18n:scada.symbol.dash}", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 0, "step": 1, @@ -456,14 +429,14 @@ }, { "id": "flowStyleGap", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "subLabel": "{i18n:scada.symbol.gap}", "fieldSuffix": "px", - "min": 1, + "min": 0, "step": 1, "disabled": false, "visible": true @@ -471,7 +444,7 @@ { "id": "flowDashCap", "name": "{i18n:scada.symbol.flow-dash-cap}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "select", "default": "butt", "items": [ diff --git a/application/src/main/data/json/system/scada_symbols/cross-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/cross-connector-hp.svg index 480666a589..db6b091e0e 100644 --- a/application/src/main/data/json/system/scada_symbols/cross-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/cross-connector-hp.svg @@ -7,7 +7,7 @@ "tags": [ { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nif (ctx.properties.mainLine) {\n element.attr({'stroke-width': ctx.properties.mainLineSize});\n} else {\n element.attr({'stroke-width': ctx.properties.secondaryLineSize});\n}", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", "actions": null }, { @@ -58,8 +58,8 @@ }, { "id": "leftFlowDirection", - "name": "{i18n:scada.symbol.animation-direction}", - "hint": "{i18n:scada.symbol.animation-direction-hint}", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": "{i18n:scada.symbol.left-connector}", "type": "value", "valueType": "BOOLEAN", @@ -174,8 +174,8 @@ }, { "id": "topFlowDirection", - "name": "{i18n:scada.symbol.animation-direction}", - "hint": "{i18n:scada.symbol.animation-direction-hint}", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": "{i18n:scada.symbol.top-connector}", "type": "value", "valueType": "BOOLEAN", @@ -290,8 +290,8 @@ }, { "id": "rightFlowDirection", - "name": "{i18n:scada.symbol.flow-direction}", - "hint": "{i18n:scada.symbol.flow-direction-hint}", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": "{i18n:scada.symbol.right-connector}", "type": "value", "valueType": "BOOLEAN", @@ -406,8 +406,8 @@ }, { "id": "bottomFlowDirection", - "name": "{i18n:scada.symbol.animation-direction}", - "hint": "{i18n:scada.symbol.animation-direction-hint}", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": "{i18n:scada.symbol.bottom-connector}", "type": "value", "valueType": "BOOLEAN", @@ -483,17 +483,9 @@ } ], "properties": [ - { - "id": "mainLine", - "name": "{i18n:scada.symbol.main-line}", - "type": "switch", - "default": true, - "disabled": false, - "visible": true - }, { "id": "mainLineSize", - "name": "{i18n:scada.symbol.main-line}", + "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, "required": true, @@ -504,28 +496,11 @@ "min": 0, "max": 99, "step": 1, - "disabled": false, - "visible": true - }, - { - "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.main-line}", - "type": "number", - "default": 2, - "required": true, - "subLabel": "Secondary", - "divider": true, - "fieldSuffix": "px", - "condition": "return !model.mainLine;", - "min": 0, - "max": 99, - "step": 1, - "disabled": false, - "visible": false + "disabled": false }, { "id": "lineColor", - "name": "{i18n:scada.symbol.main-line}", + "name": "{i18n:scada.symbol.line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -533,12 +508,11 @@ }, { "id": "flowAnimationWidth", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 4, - "subLabel": "Width", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 1, "step": 1, @@ -547,8 +521,8 @@ }, { "id": "flowAnimationColor", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "color", "default": "#C8DFF7", "disabled": false, @@ -556,14 +530,14 @@ }, { "id": "flowStyleDash", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "required": true, "subLabel": "{i18n:scada.symbol.dash}", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 0, "step": 1, @@ -572,14 +546,14 @@ }, { "id": "flowStyleGap", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "subLabel": "{i18n:scada.symbol.gap}", "fieldSuffix": "px", - "min": 1, + "min": 0, "step": 1, "disabled": false, "visible": true @@ -587,7 +561,7 @@ { "id": "flowDashCap", "name": "{i18n:scada.symbol.flow-dash-cap}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "select", "default": "butt", "items": [ diff --git a/application/src/main/data/json/system/scada_symbols/horizontal-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/horizontal-connector-hp.svg index 230c7b68a5..93ef81f57b 100644 --- a/application/src/main/data/json/system/scada_symbols/horizontal-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/horizontal-connector-hp.svg @@ -12,7 +12,7 @@ }, { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nif (ctx.properties.mainLine) {\n element.attr({'stroke-width': ctx.properties.mainLineSize});\n} else {\n element.attr({'stroke-width': ctx.properties.secondaryLineSize});\n}", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", "actions": null } ], @@ -53,18 +53,18 @@ "defaultWidgetActionSettings": null }, { - "id": "arrowDirection", - "name": "{i18n:scada.symbol.arrow-direction}", - "hint": "{i18n:scada.symbol.arrow-direction-hint}", + "id": "flowAnimation", + "name": "{i18n:scada.symbol.flow-animation}", + "hint": "{i18n:scada.symbol.flow-animation-hint}", "group": null, "type": "value", "valueType": "BOOLEAN", - "trueLabel": "{i18n:scada.symbol.forward}", - "falseLabel": "{i18n:scada.symbol.reverse}", - "stateLabel": "{i18n:scada.symbol.forward}", + "trueLabel": "{i18n:scada.symbol.present}", + "falseLabel": "{i18n:scada.symbol.absent}", + "stateLabel": "{i18n:scada.symbol.flow-present}", "defaultGetValueSettings": { "action": "DO_NOTHING", - "defaultValue": true, + "defaultValue": false, "executeRpc": { "method": "getState", "requestTimeout": 5000, @@ -72,34 +72,38 @@ "persistentPollingInterval": 1000 }, "getAttribute": { - "scope": null, - "key": "state" + "key": "state", + "scope": null }, "getTimeSeries": { "key": "state" }, + "getAlarmStatus": { + "severityList": null, + "typeList": null + }, "dataToValue": { "type": "NONE", - "dataToValueFunction": "/* Should return boolean value */\nreturn data;", - "compareToValue": true + "compareToValue": true, + "dataToValueFunction": "/* Should return boolean value */\nreturn data;" } }, "defaultSetValueSettings": null, "defaultWidgetActionSettings": null }, { - "id": "flowAnimation", - "name": "{i18n:scada.symbol.flow-animation}", - "hint": "{i18n:scada.symbol.flow-animation-hint}", + "id": "arrowDirection", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": null, "type": "value", "valueType": "BOOLEAN", - "trueLabel": "{i18n:scada.symbol.present}", - "falseLabel": "{i18n:scada.symbol.absent}", - "stateLabel": "{i18n:scada.symbol.flow-present}", + "trueLabel": "{i18n:scada.symbol.forward}", + "falseLabel": "{i18n:scada.symbol.reverse}", + "stateLabel": "{i18n:scada.symbol.forward}", "defaultGetValueSettings": { "action": "DO_NOTHING", - "defaultValue": false, + "defaultValue": true, "executeRpc": { "method": "getState", "requestTimeout": 5000, @@ -107,20 +111,16 @@ "persistentPollingInterval": 1000 }, "getAttribute": { - "key": "state", - "scope": null + "scope": null, + "key": "state" }, "getTimeSeries": { "key": "state" }, - "getAlarmStatus": { - "severityList": null, - "typeList": null - }, "dataToValue": { "type": "NONE", - "compareToValue": true, - "dataToValueFunction": "/* Should return boolean value */\nreturn data;" + "dataToValueFunction": "/* Should return boolean value */\nreturn data;", + "compareToValue": true } }, "defaultSetValueSettings": null, @@ -167,48 +167,22 @@ ], "properties": [ { - "id": "mainLine", - "name": "{i18n:scada.symbol.main-line}", - "type": "switch", - "default": true, - "disabled": false, - "visible": true - }, - { - "id": "mainLineSize", - "name": "{i18n:scada.symbol.main-line}", + "id": "lineSize", + "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, "required": true, - "subLabel": "Main", - "divider": true, + "divider": false, "fieldSuffix": "px", - "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, "visible": true }, - { - "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.main-line}", - "type": "number", - "default": 2, - "required": true, - "subLabel": "Secondary", - "divider": true, - "fieldSuffix": "px", - "condition": "return !model.mainLine;", - "min": 0, - "max": 99, - "step": 1, - "disabled": false, - "visible": false - }, { "id": "lineColor", - "name": "{i18n:scada.symbol.main-line}", + "name": "{i18n:scada.symbol.line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -216,11 +190,10 @@ }, { "id": "flowAnimationWidth", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 4, - "subLabel": "Width", "fieldSuffix": "px", "min": 1, "step": 1, @@ -229,8 +202,8 @@ }, { "id": "flowAnimationColor", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "color", "default": "#C8DFF7", "disabled": false, @@ -238,14 +211,14 @@ }, { "id": "flowStyleDash", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "required": true, "subLabel": "{i18n:scada.symbol.dash}", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 0, "step": 1, @@ -254,14 +227,14 @@ }, { "id": "flowStyleGap", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "subLabel": "{i18n:scada.symbol.gap}", "fieldSuffix": "px", - "min": 1, + "min": 0, "step": 1, "disabled": false, "visible": true @@ -269,7 +242,7 @@ { "id": "flowDashCap", "name": "{i18n:scada.symbol.flow-dash-cap}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "select", "default": "butt", "items": [ diff --git a/application/src/main/data/json/system/scada_symbols/left-bottom-elbow-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/left-bottom-elbow-connector-hp.svg index 1dbeec5f92..1c24cca2e0 100644 --- a/application/src/main/data/json/system/scada_symbols/left-bottom-elbow-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/left-bottom-elbow-connector-hp.svg @@ -7,7 +7,7 @@ "tags": [ { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nif (ctx.properties.mainLine) {\n element.attr({'stroke-width': ctx.properties.mainLineSize});\n} else {\n element.attr({'stroke-width': ctx.properties.secondaryLineSize});\n}", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", "actions": null } ], @@ -32,8 +32,8 @@ "persistentPollingInterval": 1000 }, "getAttribute": { - "key": "state", - "scope": null + "scope": "SHARED_SCOPE", + "key": "flow" }, "getTimeSeries": { "key": "state" @@ -44,8 +44,8 @@ }, "dataToValue": { "type": "NONE", - "compareToValue": true, - "dataToValueFunction": "/* Should return boolean value */\nreturn data;" + "dataToValueFunction": "/* Should return boolean value */\nreturn data;", + "compareToValue": true } }, "defaultSetValueSettings": null, @@ -53,8 +53,8 @@ }, { "id": "animationDirection", - "name": "{i18n:scada.symbol.animation-direction}", - "hint": "{i18n:scada.symbol.animation-direction-hint}", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": null, "type": "value", "valueType": "BOOLEAN", @@ -131,49 +131,22 @@ ], "properties": [ { - "id": "mainLine", - "name": "{i18n:scada.symbol.main-line}", - "type": "switch", - "default": true, - "disabled": false, - "visible": true - }, - { - "id": "mainLineSize", - "name": "{i18n:scada.symbol.main-line}", + "id": "lineSize", + "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, "required": true, - "subLabel": "Main", - "divider": true, + "divider": false, "fieldSuffix": "px", - "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, "visible": true }, - { - "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.main-line}", - "type": "number", - "default": 2, - "required": true, - "subLabel": "Secondary", - "divider": true, - "fieldSuffix": "px", - "disableOnProperty": "mainLine", - "condition": "return !model.mainLine;", - "min": 0, - "max": 99, - "step": 1, - "disabled": false, - "visible": false - }, { "id": "lineColor", - "name": "{i18n:scada.symbol.main-line}", + "name": "{i18n:scada.symbol.line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -181,12 +154,11 @@ }, { "id": "flowAnimationWidth", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 4, - "subLabel": "Width", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 1, "step": 1, @@ -195,8 +167,8 @@ }, { "id": "flowAnimationColor", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "color", "default": "#C8DFF7", "disabled": false, @@ -204,14 +176,14 @@ }, { "id": "flowStyleDash", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "required": true, "subLabel": "{i18n:scada.symbol.dash}", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 0, "step": 1, @@ -220,14 +192,14 @@ }, { "id": "flowStyleGap", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "subLabel": "{i18n:scada.symbol.gap}", "fieldSuffix": "px", - "min": 1, + "min": 0, "step": 1, "disabled": false, "visible": true @@ -235,7 +207,7 @@ { "id": "flowDashCap", "name": "{i18n:scada.symbol.flow-dash-cap}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "select", "default": "butt", "items": [ diff --git a/application/src/main/data/json/system/scada_symbols/left-tee-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/left-tee-connector-hp.svg index 82b8babcf7..461e91fe94 100644 --- a/application/src/main/data/json/system/scada_symbols/left-tee-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/left-tee-connector-hp.svg @@ -7,7 +7,7 @@ "tags": [ { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nif (ctx.properties.mainLine) {\n element.attr({'stroke-width': ctx.properties.mainLineSize});\n} else {\n element.attr({'stroke-width': ctx.properties.secondaryLineSize});\n}", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", "actions": null }, { @@ -58,8 +58,8 @@ }, { "id": "leftFlowDirection", - "name": "{i18n:scada.symbol.animation-direction}", - "hint": "{i18n:scada.symbol.animation-direction-hint}", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": "{i18n:scada.symbol.left-connector}", "type": "value", "valueType": "BOOLEAN", @@ -174,8 +174,8 @@ }, { "id": "topFlowDirection", - "name": "{i18n:scada.symbol.animation-direction}", - "hint": "{i18n:scada.symbol.animation-direction-hint}", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": "{i18n:scada.symbol.top-connector}", "type": "value", "valueType": "BOOLEAN", @@ -290,8 +290,8 @@ }, { "id": "bottomFlowDirection", - "name": "{i18n:scada.symbol.flow-direction}", - "hint": "{i18n:scada.symbol.flow-direction-hint}", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": "{i18n:scada.symbol.bottom-connector}", "type": "value", "valueType": "BOOLEAN", @@ -368,48 +368,22 @@ ], "properties": [ { - "id": "mainLine", - "name": "{i18n:scada.symbol.main-line}", - "type": "switch", - "default": true, - "disabled": false, - "visible": true - }, - { - "id": "mainLineSize", - "name": "{i18n:scada.symbol.main-line}", + "id": "lineSize", + "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, "required": true, - "subLabel": "Main", - "divider": true, + "divider": false, "fieldSuffix": "px", - "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, "visible": true }, - { - "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.main-line}", - "type": "number", - "default": 2, - "required": true, - "subLabel": "Secondary", - "divider": true, - "fieldSuffix": "px", - "condition": "return !model.mainLine;", - "min": 0, - "max": 99, - "step": 1, - "disabled": false, - "visible": false - }, { "id": "lineColor", - "name": "{i18n:scada.symbol.main-line}", + "name": "{i18n:scada.symbol.line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -417,12 +391,11 @@ }, { "id": "flowAnimationWidth", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 4, - "subLabel": "Width", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 1, "step": 1, @@ -431,8 +404,8 @@ }, { "id": "flowAnimationColor", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "color", "default": "#C8DFF7", "disabled": false, @@ -440,14 +413,14 @@ }, { "id": "flowStyleDash", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "required": true, "subLabel": "{i18n:scada.symbol.dash}", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 0, "step": 1, @@ -456,14 +429,14 @@ }, { "id": "flowStyleGap", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "subLabel": "{i18n:scada.symbol.gap}", "fieldSuffix": "px", - "min": 1, + "min": 0, "step": 1, "disabled": false, "visible": true @@ -471,7 +444,7 @@ { "id": "flowDashCap", "name": "{i18n:scada.symbol.flow-dash-cap}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "select", "default": "butt", "items": [ diff --git a/application/src/main/data/json/system/scada_symbols/left-top-elbow-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/left-top-elbow-connector-hp.svg index 3c811a2372..19e217a428 100644 --- a/application/src/main/data/json/system/scada_symbols/left-top-elbow-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/left-top-elbow-connector-hp.svg @@ -7,7 +7,7 @@ "tags": [ { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nif (ctx.properties.mainLine) {\n element.attr({'stroke-width': ctx.properties.mainLineSize});\n} else {\n element.attr({'stroke-width': ctx.properties.secondaryLineSize});\n}", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", "actions": null } ], @@ -53,8 +53,8 @@ }, { "id": "animationDirection", - "name": "{i18n:scada.symbol.animation-direction}", - "hint": "{i18n:scada.symbol.animation-direction-hint}", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": null, "type": "value", "valueType": "BOOLEAN", @@ -131,48 +131,22 @@ ], "properties": [ { - "id": "mainLine", - "name": "{i18n:scada.symbol.main-line}", - "type": "switch", - "default": true, - "disabled": false, - "visible": true - }, - { - "id": "mainLineSize", - "name": "{i18n:scada.symbol.main-line}", + "id": "lineSize", + "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, "required": true, - "subLabel": "Main", - "divider": true, + "divider": false, "fieldSuffix": "px", - "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, "visible": true }, - { - "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.main-line}", - "type": "number", - "default": 2, - "required": true, - "subLabel": "Secondary", - "divider": true, - "fieldSuffix": "px", - "condition": "return !model.mainLine;", - "min": 0, - "max": 99, - "step": 1, - "disabled": false, - "visible": false - }, { "id": "lineColor", - "name": "{i18n:scada.symbol.main-line}", + "name": "{i18n:scada.symbol.line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -180,12 +154,11 @@ }, { "id": "flowAnimationWidth", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 4, - "subLabel": "Width", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 1, "step": 1, @@ -194,8 +167,8 @@ }, { "id": "flowAnimationColor", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "color", "default": "#C8DFF7", "disabled": false, @@ -203,14 +176,14 @@ }, { "id": "flowStyleDash", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "required": true, "subLabel": "{i18n:scada.symbol.dash}", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 0, "step": 1, @@ -219,14 +192,14 @@ }, { "id": "flowStyleGap", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "subLabel": "{i18n:scada.symbol.gap}", "fieldSuffix": "px", - "min": 1, + "min": 0, "step": 1, "disabled": false, "visible": true @@ -234,7 +207,7 @@ { "id": "flowDashCap", "name": "{i18n:scada.symbol.flow-dash-cap}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "select", "default": "butt", "items": [ diff --git a/application/src/main/data/json/system/scada_symbols/long-horizontal-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/long-horizontal-connector-hp.svg index 3a3296544d..0fe805c9a7 100644 --- a/application/src/main/data/json/system/scada_symbols/long-horizontal-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/long-horizontal-connector-hp.svg @@ -12,7 +12,7 @@ }, { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nif (ctx.properties.mainLine) {\n element.attr({'stroke-width': ctx.properties.mainLineSize});\n} else {\n element.attr({'stroke-width': ctx.properties.secondaryLineSize});\n}", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", "actions": null } ], @@ -53,18 +53,18 @@ "defaultWidgetActionSettings": null }, { - "id": "arrowDirection", - "name": "{i18n:scada.symbol.arrow-direction}", - "hint": "{i18n:scada.symbol.arrow-direction-hint}", + "id": "flowAnimation", + "name": "{i18n:scada.symbol.flow-animation}", + "hint": "{i18n:scada.symbol.flow-animation-hint}", "group": null, "type": "value", "valueType": "BOOLEAN", - "trueLabel": "{i18n:scada.symbol.forward}", - "falseLabel": "{i18n:scada.symbol.reverse}", - "stateLabel": "{i18n:scada.symbol.forward}", + "trueLabel": "{i18n:scada.symbol.present}", + "falseLabel": "{i18n:scada.symbol.absent}", + "stateLabel": "{i18n:scada.symbol.flow-present}", "defaultGetValueSettings": { "action": "DO_NOTHING", - "defaultValue": true, + "defaultValue": false, "executeRpc": { "method": "getState", "requestTimeout": 5000, @@ -72,34 +72,38 @@ "persistentPollingInterval": 1000 }, "getAttribute": { - "scope": null, - "key": "state" + "key": "state", + "scope": null }, "getTimeSeries": { "key": "state" }, + "getAlarmStatus": { + "severityList": null, + "typeList": null + }, "dataToValue": { "type": "NONE", - "dataToValueFunction": "/* Should return boolean value */\nreturn data;", - "compareToValue": true + "compareToValue": true, + "dataToValueFunction": "/* Should return boolean value */\nreturn data;" } }, "defaultSetValueSettings": null, "defaultWidgetActionSettings": null }, { - "id": "flowAnimation", - "name": "{i18n:scada.symbol.flow-animation}", - "hint": "{i18n:scada.symbol.flow-animation-hint}", + "id": "arrowDirection", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": null, "type": "value", "valueType": "BOOLEAN", - "trueLabel": "{i18n:scada.symbol.present}", - "falseLabel": "{i18n:scada.symbol.absent}", - "stateLabel": "{i18n:scada.symbol.flow-present}", + "trueLabel": "{i18n:scada.symbol.forward}", + "falseLabel": "{i18n:scada.symbol.reverse}", + "stateLabel": "{i18n:scada.symbol.forward}", "defaultGetValueSettings": { "action": "DO_NOTHING", - "defaultValue": false, + "defaultValue": true, "executeRpc": { "method": "getState", "requestTimeout": 5000, @@ -107,20 +111,16 @@ "persistentPollingInterval": 1000 }, "getAttribute": { - "key": "state", - "scope": null + "scope": null, + "key": "state" }, "getTimeSeries": { "key": "state" }, - "getAlarmStatus": { - "severityList": null, - "typeList": null - }, "dataToValue": { "type": "NONE", - "compareToValue": true, - "dataToValueFunction": "/* Should return boolean value */\nreturn data;" + "dataToValueFunction": "/* Should return boolean value */\nreturn data;", + "compareToValue": true } }, "defaultSetValueSettings": null, @@ -167,39 +167,13 @@ ], "properties": [ { - "id": "mainLine", - "name": "{i18n:scada.symbol.main-line}", - "type": "switch", - "default": true, - "disabled": false, - "visible": true - }, - { - "id": "mainLineSize", - "name": "{i18n:scada.symbol.main-line}", + "id": "lineSize", + "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, "required": true, - "subLabel": "Main", - "divider": true, - "fieldSuffix": "px", - "condition": "return model.mainLine;", - "min": 0, - "max": 99, - "step": 1, - "disabled": false, - "visible": false - }, - { - "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.main-line}", - "type": "number", - "default": 2, - "required": true, - "subLabel": "Secondary", - "divider": true, + "divider": false, "fieldSuffix": "px", - "condition": "return !model.mainLine;", "min": 0, "max": 99, "step": 1, @@ -208,7 +182,7 @@ }, { "id": "lineColor", - "name": "{i18n:scada.symbol.main-line}", + "name": "{i18n:scada.symbol.line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -216,12 +190,11 @@ }, { "id": "flowAnimationWidth", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 4, - "subLabel": "Width", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 1, "step": 1, @@ -230,8 +203,8 @@ }, { "id": "flowAnimationColor", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "color", "default": "#C8DFF7", "disabled": false, @@ -239,14 +212,14 @@ }, { "id": "flowStyleDash", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "required": true, "subLabel": "{i18n:scada.symbol.dash}", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 0, "step": 1, @@ -255,14 +228,14 @@ }, { "id": "flowStyleGap", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "subLabel": "{i18n:scada.symbol.gap}", "fieldSuffix": "px", - "min": 1, + "min": 0, "step": 1, "disabled": false, "visible": true @@ -270,7 +243,7 @@ { "id": "flowDashCap", "name": "{i18n:scada.symbol.flow-dash-cap}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "select", "default": "butt", "items": [ diff --git a/application/src/main/data/json/system/scada_symbols/long-vertical-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/long-vertical-connector-hp.svg index e66fd62c2e..411859b5ed 100644 --- a/application/src/main/data/json/system/scada_symbols/long-vertical-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/long-vertical-connector-hp.svg @@ -12,7 +12,7 @@ }, { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nif (ctx.properties.mainLine) {\n element.attr({'stroke-width': ctx.properties.mainLineSize});\n} else {\n element.attr({'stroke-width': ctx.properties.secondaryLineSize});\n}", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", "actions": null } ], @@ -53,18 +53,18 @@ "defaultWidgetActionSettings": null }, { - "id": "arrowDirection", - "name": "{i18n:scada.symbol.arrow-direction}", - "hint": "{i18n:scada.symbol.arrow-direction-hint}", + "id": "flowAnimation", + "name": "{i18n:scada.symbol.flow-animation}", + "hint": "{i18n:scada.symbol.flow-animation-hint}", "group": null, "type": "value", "valueType": "BOOLEAN", - "trueLabel": "{i18n:scada.symbol.forward}", - "falseLabel": "{i18n:scada.symbol.reverse}", - "stateLabel": "{i18n:scada.symbol.forward}", + "trueLabel": "{i18n:scada.symbol.present}", + "falseLabel": "{i18n:scada.symbol.absent}", + "stateLabel": "{i18n:scada.symbol.flow-present}", "defaultGetValueSettings": { "action": "DO_NOTHING", - "defaultValue": true, + "defaultValue": false, "executeRpc": { "method": "getState", "requestTimeout": 5000, @@ -72,34 +72,38 @@ "persistentPollingInterval": 1000 }, "getAttribute": { - "scope": null, - "key": "state" + "key": "state", + "scope": null }, "getTimeSeries": { "key": "state" }, + "getAlarmStatus": { + "severityList": null, + "typeList": null + }, "dataToValue": { "type": "NONE", - "dataToValueFunction": "/* Should return boolean value */\nreturn data;", - "compareToValue": true + "compareToValue": true, + "dataToValueFunction": "/* Should return boolean value */\nreturn data;" } }, "defaultSetValueSettings": null, "defaultWidgetActionSettings": null }, { - "id": "flowAnimation", - "name": "{i18n:scada.symbol.flow-animation}", - "hint": "{i18n:scada.symbol.flow-animation-hint}", + "id": "arrowDirection", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": null, "type": "value", "valueType": "BOOLEAN", - "trueLabel": "{i18n:scada.symbol.present}", - "falseLabel": "{i18n:scada.symbol.absent}", - "stateLabel": "{i18n:scada.symbol.flow-present}", + "trueLabel": "{i18n:scada.symbol.forward}", + "falseLabel": "{i18n:scada.symbol.reverse}", + "stateLabel": "{i18n:scada.symbol.forward}", "defaultGetValueSettings": { "action": "DO_NOTHING", - "defaultValue": false, + "defaultValue": true, "executeRpc": { "method": "getState", "requestTimeout": 5000, @@ -107,20 +111,16 @@ "persistentPollingInterval": 1000 }, "getAttribute": { - "key": "state", - "scope": null + "scope": null, + "key": "state" }, "getTimeSeries": { "key": "state" }, - "getAlarmStatus": { - "severityList": null, - "typeList": null - }, "dataToValue": { "type": "NONE", - "compareToValue": true, - "dataToValueFunction": "/* Should return boolean value */\nreturn data;" + "dataToValueFunction": "/* Should return boolean value */\nreturn data;", + "compareToValue": true } }, "defaultSetValueSettings": null, @@ -167,48 +167,22 @@ ], "properties": [ { - "id": "mainLine", - "name": "{i18n:scada.symbol.main-line}", - "type": "switch", - "default": true, - "disabled": false, - "visible": true - }, - { - "id": "mainLineSize", - "name": "{i18n:scada.symbol.main-line}", + "id": "lineSize", + "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, "required": true, - "subLabel": "Main", - "divider": true, + "divider": false, "fieldSuffix": "px", - "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, "visible": true }, - { - "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.main-line}", - "type": "number", - "default": 2, - "required": true, - "subLabel": "Secondary", - "divider": true, - "fieldSuffix": "px", - "condition": "return !model.mainLine;", - "min": 0, - "max": 99, - "step": 1, - "disabled": false, - "visible": false - }, { "id": "lineColor", - "name": "{i18n:scada.symbol.main-line}", + "name": "{i18n:scada.symbol.line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -216,12 +190,11 @@ }, { "id": "flowAnimationWidth", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 4, - "subLabel": "Width", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 1, "step": 1, @@ -230,8 +203,8 @@ }, { "id": "flowAnimationColor", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "color", "default": "#C8DFF7", "disabled": false, @@ -239,14 +212,14 @@ }, { "id": "flowStyleDash", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "required": true, "subLabel": "{i18n:scada.symbol.dash}", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 0, "step": 1, @@ -255,22 +228,20 @@ }, { "id": "flowStyleGap", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "subLabel": "{i18n:scada.symbol.gap}", "fieldSuffix": "px", - "min": 1, - "step": 1, - "disabled": false, - "visible": true + "min": 0, + "step": 1 }, { "id": "flowDashCap", "name": "{i18n:scada.symbol.flow-dash-cap}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "select", "default": "butt", "items": [ diff --git a/application/src/main/data/json/system/scada_symbols/right-tee-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/right-tee-connector-hp.svg index e40936c152..dafcf53647 100644 --- a/application/src/main/data/json/system/scada_symbols/right-tee-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/right-tee-connector-hp.svg @@ -7,7 +7,7 @@ "tags": [ { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nif (ctx.properties.mainLine) {\n element.attr({'stroke-width': ctx.properties.mainLineSize});\n} else {\n element.attr({'stroke-width': ctx.properties.secondaryLineSize});\n}", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", "actions": null }, { @@ -58,8 +58,8 @@ }, { "id": "topFlowDirection", - "name": "{i18n:scada.symbol.animation-direction}", - "hint": "{i18n:scada.symbol.animation-direction-hint}", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": "{i18n:scada.symbol.top-connector}", "type": "value", "valueType": "BOOLEAN", @@ -174,8 +174,8 @@ }, { "id": "rightFlowDirection", - "name": "{i18n:scada.symbol.flow-direction}", - "hint": "{i18n:scada.symbol.flow-direction-hint}", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": "{i18n:scada.symbol.right-connector}", "type": "value", "valueType": "BOOLEAN", @@ -290,8 +290,8 @@ }, { "id": "bottomFlowDirection", - "name": "{i18n:scada.symbol.animation-direction}", - "hint": "{i18n:scada.symbol.animation-direction-hint}", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": "{i18n:scada.symbol.bottom-connector}", "type": "value", "valueType": "BOOLEAN", @@ -368,48 +368,22 @@ ], "properties": [ { - "id": "mainLine", - "name": "{i18n:scada.symbol.main-line}", - "type": "switch", - "default": true, - "disabled": false, - "visible": true - }, - { - "id": "mainLineSize", - "name": "{i18n:scada.symbol.main-line}", + "id": "lineSize", + "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, "required": true, - "subLabel": "Main", - "divider": true, + "divider": false, "fieldSuffix": "px", - "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, "visible": true }, - { - "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.main-line}", - "type": "number", - "default": 2, - "required": true, - "subLabel": "Secondary", - "divider": true, - "fieldSuffix": "px", - "condition": "return !model.mainLine;", - "min": 0, - "max": 99, - "step": 1, - "disabled": false, - "visible": false - }, { "id": "lineColor", - "name": "{i18n:scada.symbol.main-line}", + "name": "{i18n:scada.symbol.line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -417,12 +391,11 @@ }, { "id": "flowAnimationWidth", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 4, - "subLabel": "Width", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 1, "step": 1, @@ -431,8 +404,8 @@ }, { "id": "flowAnimationColor", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "color", "default": "#C8DFF7", "disabled": false, @@ -440,14 +413,14 @@ }, { "id": "flowStyleDash", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "required": true, "subLabel": "{i18n:scada.symbol.dash}", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 0, "step": 1, @@ -456,14 +429,14 @@ }, { "id": "flowStyleGap", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "subLabel": "{i18n:scada.symbol.gap}", "fieldSuffix": "px", - "min": 1, + "min": 0, "step": 1, "disabled": false, "visible": true @@ -471,7 +444,7 @@ { "id": "flowDashCap", "name": "{i18n:scada.symbol.flow-dash-cap}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "select", "default": "butt", "items": [ diff --git a/application/src/main/data/json/system/scada_symbols/top-right-elbow-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/top-right-elbow-connector-hp.svg index 8c5224c9f8..f9f2fe20d2 100644 --- a/application/src/main/data/json/system/scada_symbols/top-right-elbow-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/top-right-elbow-connector-hp.svg @@ -7,7 +7,7 @@ "tags": [ { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nif (ctx.properties.mainLine) {\n element.attr({'stroke-width': ctx.properties.mainLineSize});\n} else {\n element.attr({'stroke-width': ctx.properties.secondaryLineSize});\n}", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", "actions": null } ], @@ -53,8 +53,8 @@ }, { "id": "animationDirection", - "name": "{i18n:scada.symbol.animation-direction}", - "hint": "{i18n:scada.symbol.animation-direction-hint}", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": null, "type": "value", "valueType": "BOOLEAN", @@ -131,48 +131,22 @@ ], "properties": [ { - "id": "mainLine", - "name": "{i18n:scada.symbol.main-line}", - "type": "switch", - "default": true, - "disabled": false, - "visible": true - }, - { - "id": "mainLineSize", - "name": "{i18n:scada.symbol.main-line}", + "id": "lineSize", + "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, "required": true, - "subLabel": "Main", - "divider": true, + "divider": false, "fieldSuffix": "px", - "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, "visible": true }, - { - "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.main-line}", - "type": "number", - "default": 2, - "required": true, - "subLabel": "Secondary", - "divider": true, - "fieldSuffix": "px", - "condition": "return !model.mainLine;", - "min": 0, - "max": 99, - "step": 1, - "disabled": false, - "visible": false - }, { "id": "lineColor", - "name": "{i18n:scada.symbol.main-line}", + "name": "{i18n:scada.symbol.line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -180,12 +154,11 @@ }, { "id": "flowAnimationWidth", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 4, - "subLabel": "Width", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 1, "step": 1, @@ -194,8 +167,8 @@ }, { "id": "flowAnimationColor", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "color", "default": "#C8DFF7", "disabled": false, @@ -203,14 +176,14 @@ }, { "id": "flowStyleDash", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "required": true, "subLabel": "{i18n:scada.symbol.dash}", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 0, "step": 1, @@ -219,14 +192,14 @@ }, { "id": "flowStyleGap", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "subLabel": "{i18n:scada.symbol.gap}", "fieldSuffix": "px", - "min": 1, + "min": 0, "step": 1, "disabled": false, "visible": true @@ -234,7 +207,7 @@ { "id": "flowDashCap", "name": "{i18n:scada.symbol.flow-dash-cap}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "select", "default": "butt", "items": [ diff --git a/application/src/main/data/json/system/scada_symbols/top-tee-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/top-tee-connector-hp.svg index 101799fd4b..f61bf85ccd 100644 --- a/application/src/main/data/json/system/scada_symbols/top-tee-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/top-tee-connector-hp.svg @@ -7,7 +7,7 @@ "tags": [ { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nif (ctx.properties.mainLine) {\n element.attr({'stroke-width': ctx.properties.mainLineSize});\n} else {\n element.attr({'stroke-width': ctx.properties.secondaryLineSize});\n}", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", "actions": null }, { @@ -58,8 +58,8 @@ }, { "id": "leftFlowDirection", - "name": "{i18n:scada.symbol.animation-direction}", - "hint": "{i18n:scada.symbol.animation-direction-hint}", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": "{i18n:scada.symbol.left-connector}", "type": "value", "valueType": "BOOLEAN", @@ -174,8 +174,8 @@ }, { "id": "topFlowDirection", - "name": "{i18n:scada.symbol.animation-direction}", - "hint": "{i18n:scada.symbol.animation-direction-hint}", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": "{i18n:scada.symbol.top-connector}", "type": "value", "valueType": "BOOLEAN", @@ -290,8 +290,8 @@ }, { "id": "rightFlowDirection", - "name": "{i18n:scada.symbol.flow-direction}", - "hint": "{i18n:scada.symbol.flow-direction-hint}", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": "{i18n:scada.symbol.right-connector}", "type": "value", "valueType": "BOOLEAN", @@ -368,48 +368,22 @@ ], "properties": [ { - "id": "mainLine", - "name": "{i18n:scada.symbol.main-line}", - "type": "switch", - "default": true, - "disabled": false, - "visible": true - }, - { - "id": "mainLineSize", - "name": "{i18n:scada.symbol.main-line}", + "id": "lineSize", + "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, "required": true, - "subLabel": "Main", - "divider": true, + "divider": false, "fieldSuffix": "px", - "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, "visible": true }, - { - "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.main-line}", - "type": "number", - "default": 2, - "required": true, - "subLabel": "Secondary", - "divider": true, - "fieldSuffix": "px", - "condition": "return !model.mainLine;", - "min": 0, - "max": 99, - "step": 1, - "disabled": false, - "visible": false - }, { "id": "lineColor", - "name": "{i18n:scada.symbol.main-line}", + "name": "{i18n:scada.symbol.line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -417,12 +391,11 @@ }, { "id": "flowAnimationWidth", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 4, - "subLabel": "Width", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 1, "step": 1, @@ -431,8 +404,8 @@ }, { "id": "flowAnimationColor", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "color", "default": "#C8DFF7", "disabled": false, @@ -440,14 +413,14 @@ }, { "id": "flowStyleDash", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "required": true, "subLabel": "{i18n:scada.symbol.dash}", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 0, "step": 1, @@ -456,14 +429,14 @@ }, { "id": "flowStyleGap", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "subLabel": "{i18n:scada.symbol.gap}", "fieldSuffix": "px", - "min": 1, + "min": 0, "step": 1, "disabled": false, "visible": true @@ -471,7 +444,7 @@ { "id": "flowDashCap", "name": "{i18n:scada.symbol.flow-dash-cap}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "select", "default": "butt", "items": [ diff --git a/application/src/main/data/json/system/scada_symbols/vertical-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/vertical-connector-hp.svg index 698a910f8d..9667928e4e 100644 --- a/application/src/main/data/json/system/scada_symbols/vertical-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/vertical-connector-hp.svg @@ -12,7 +12,7 @@ }, { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nif (ctx.properties.mainLine) {\n element.attr({'stroke-width': ctx.properties.mainLineSize});\n} else {\n element.attr({'stroke-width': ctx.properties.secondaryLineSize});\n}", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", "actions": null } ], @@ -53,18 +53,18 @@ "defaultWidgetActionSettings": null }, { - "id": "arrowDirection", - "name": "{i18n:scada.symbol.arrow-direction}", - "hint": "{i18n:scada.symbol.arrow-direction-hint}", + "id": "flowAnimation", + "name": "{i18n:scada.symbol.flow-animation}", + "hint": "{i18n:scada.symbol.flow-animation-hint}", "group": null, "type": "value", "valueType": "BOOLEAN", - "trueLabel": "{i18n:scada.symbol.forward}", - "falseLabel": "{i18n:scada.symbol.reverse}", - "stateLabel": "{i18n:scada.symbol.forward}", + "trueLabel": "{i18n:scada.symbol.present}", + "falseLabel": "{i18n:scada.symbol.absent}", + "stateLabel": "{i18n:scada.symbol.flow-present}", "defaultGetValueSettings": { "action": "DO_NOTHING", - "defaultValue": true, + "defaultValue": false, "executeRpc": { "method": "getState", "requestTimeout": 5000, @@ -72,34 +72,38 @@ "persistentPollingInterval": 1000 }, "getAttribute": { - "scope": null, - "key": "state" + "key": "state", + "scope": null }, "getTimeSeries": { "key": "state" }, + "getAlarmStatus": { + "severityList": null, + "typeList": null + }, "dataToValue": { "type": "NONE", - "dataToValueFunction": "/* Should return boolean value */\nreturn data;", - "compareToValue": true + "compareToValue": true, + "dataToValueFunction": "/* Should return boolean value */\nreturn data;" } }, "defaultSetValueSettings": null, "defaultWidgetActionSettings": null }, { - "id": "flowAnimation", - "name": "{i18n:scada.symbol.flow-animation}", - "hint": "{i18n:scada.symbol.flow-animation-hint}", + "id": "arrowDirection", + "name": "{i18n:scada.symbol.arrow-direction}", + "hint": "{i18n:scada.symbol.arrow-direction-hint}", "group": null, "type": "value", "valueType": "BOOLEAN", - "trueLabel": "{i18n:scada.symbol.present}", - "falseLabel": "{i18n:scada.symbol.absent}", - "stateLabel": "{i18n:scada.symbol.flow-present}", + "trueLabel": "{i18n:scada.symbol.forward}", + "falseLabel": "{i18n:scada.symbol.reverse}", + "stateLabel": "{i18n:scada.symbol.forward}", "defaultGetValueSettings": { "action": "DO_NOTHING", - "defaultValue": false, + "defaultValue": true, "executeRpc": { "method": "getState", "requestTimeout": 5000, @@ -107,20 +111,16 @@ "persistentPollingInterval": 1000 }, "getAttribute": { - "key": "state", - "scope": null + "scope": null, + "key": "state" }, "getTimeSeries": { "key": "state" }, - "getAlarmStatus": { - "severityList": null, - "typeList": null - }, "dataToValue": { "type": "NONE", - "compareToValue": true, - "dataToValueFunction": "/* Should return boolean value */\nreturn data;" + "dataToValueFunction": "/* Should return boolean value */\nreturn data;", + "compareToValue": true } }, "defaultSetValueSettings": null, @@ -167,48 +167,22 @@ ], "properties": [ { - "id": "mainLine", - "name": "{i18n:scada.symbol.main-line}", - "type": "switch", - "default": true, - "disabled": false, - "visible": true - }, - { - "id": "mainLineSize", - "name": "{i18n:scada.symbol.main-line}", + "id": "lineSize", + "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, "required": true, - "subLabel": "Main", - "divider": true, + "divider": false, "fieldSuffix": "px", - "condition": "return model.mainLine;", "min": 0, "max": 99, "step": 1, "disabled": false, "visible": true }, - { - "id": "secondaryLineSize", - "name": "{i18n:scada.symbol.main-line}", - "type": "number", - "default": 2, - "required": true, - "subLabel": "Secondary", - "divider": true, - "fieldSuffix": "px", - "condition": "return !model.mainLine;", - "min": 0, - "max": 99, - "step": 1, - "disabled": false, - "visible": false - }, { "id": "lineColor", - "name": "{i18n:scada.symbol.main-line}", + "name": "{i18n:scada.symbol.line}", "type": "color", "default": "#1A1A1A", "disabled": false, @@ -216,12 +190,11 @@ }, { "id": "flowAnimationWidth", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 4, - "subLabel": "Width", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 1, "step": 1, @@ -230,8 +203,8 @@ }, { "id": "flowAnimationColor", - "name": "{i18n:scada.symbol.flow}", - "group": "{i18n:scada.symbol.animation}", + "name": "{i18n:scada.symbol.flow-line}", + "group": "{i18n:scada.symbol.flow}", "type": "color", "default": "#C8DFF7", "disabled": false, @@ -239,14 +212,14 @@ }, { "id": "flowStyleDash", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "required": true, "subLabel": "{i18n:scada.symbol.dash}", - "divider": true, + "divider": false, "fieldSuffix": "px", "min": 0, "step": 1, @@ -255,14 +228,14 @@ }, { "id": "flowStyleGap", - "name": "{i18n:scada.symbol.flow-style}", + "name": "{i18n:scada.symbol.flow-line-style}", "hint": "{i18n:scada.symbol.flow-style-hint}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "number", "default": 10, "subLabel": "{i18n:scada.symbol.gap}", "fieldSuffix": "px", - "min": 1, + "min": 0, "step": 1, "disabled": false, "visible": true @@ -270,7 +243,7 @@ { "id": "flowDashCap", "name": "{i18n:scada.symbol.flow-dash-cap}", - "group": "{i18n:scada.symbol.animation}", + "group": "{i18n:scada.symbol.flow}", "type": "select", "default": "butt", "items": [ diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/scada/scada-symbol.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/scada/scada-symbol.models.ts index ee63dc59a6..9e0d07fed4 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/scada/scada-symbol.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/scada/scada-symbol.models.ts @@ -190,7 +190,7 @@ const tbNamespaceRegex = //gm const tbTagRegex = /tb:tag="([^"]*)"/gms; -let syncTime = Date.now(); +const syncTime = Date.now(); const generateElementId = () => { const id = guid(); @@ -1279,9 +1279,6 @@ class FlowConnectorAnimation implements ConnectorScadaSymbolAnimation { if (!this.element.node.childElementCount) { this.element.add(this._animation); } - if (!syncTime) { - syncTime = Date.now(); - } const animateElement = this.element.node.getElementsByTagName('animate')[0]; const offset = ((Date.now() - syncTime) % 1000) * -1; (animateElement as SVGAnimationElement).beginElementAt(offset); @@ -1310,7 +1307,7 @@ class FlowConnectorAnimation implements ConnectorScadaSymbolAnimation { this._strokeLineCap = linecap; this._dashWidth = dashWidth - offset; this._dashGap = dashGap - offset; - const dashArray = `${this._dashWidth}${this._dashGap ? ` ${this._dashGap}` : ''}`; + const dashArray = `${this._dashWidth} ${this._dashGap}`; const values = `${this._dashWidth + (this._dashGap || this._dashWidth)};0`; this._animation.stroke({width, color, linecap, dasharray: dashArray}); this._animation.findOne('animate').attr('values', values); diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index a3ba654258..b9eaf3cb82 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3438,16 +3438,15 @@ "arrow-presence": "Arrow presence", "arrow-presence-hint": "Indicates whether arrow is present in connector.", "arrow-present": "Arrow present", - "arrow-direction": "Arrow/Animation direction", + "arrow-direction": "Flow direction", "arrow-direction-hint": "Indicates flow direction.", - "animation-direction": "Flow animation direction", - "animation-direction-hint": "Indicates animation flow direction.", - "flow-animation": "Flow animation", - "flow-animation-hint": "Indicates whether animation is present in connector.", + "flow-animation": "Flow presence", + "flow-animation-hint": "Indicates whether fluid is flowing in connector.", "flow": "Flow", - "flow-style": "Flow style", + "flow-line": "Line", + "flow-line-style": "Line style", "flow-style-hint": "Set the Dash and Gap values so that their sum is divisible by 100 without a remainder for perfect animation synchronization.", - "flow-dash-cap": "Flow dash cap", + "flow-dash-cap": "Dash cap", "dash-cap-butt": "Butt", "dash-cap-round": "Round", "dash-cap-square": "Square", From 8d749f593b4a14dcb23f959536612ad47c83b65a Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Thu, 27 Mar 2025 13:59:19 +0200 Subject: [PATCH 16/79] MQTT client: limit retransmission attempts to prevent unlimited memory usage and network overload --- .../server/actors/ActorSystemContext.java | 7 +- .../actors/ruleChain/DefaultTbContext.java | 9 +- ...ClientRetransmissionSettingsComponent.java | 37 +++ .../mqtt/MqttClientSettingsComponent.java | 47 ++++ .../src/main/resources/thingsboard.yml | 24 ++ .../server/msa/ContainerTestSuite.java | 12 +- .../msa/connectivity/MqttClientTest.java | 15 +- .../connectivity/MqttGatewayClientTest.java | 14 +- netty-mqtt/pom.xml | 20 ++ .../MaxRetransmissionsReachedException.java | 24 ++ .../thingsboard/mqtt/MqttChannelHandler.java | 30 +-- .../java/org/thingsboard/mqtt/MqttClient.java | 2 +- .../thingsboard/mqtt/MqttClientConfig.java | 20 ++ .../org/thingsboard/mqtt/MqttClientImpl.java | 82 ++++++- .../thingsboard/mqtt/MqttConnectResult.java | 3 + .../thingsboard/mqtt/MqttPendingPublish.java | 134 +++++++---- .../mqtt/MqttPendingSubscription.java | 119 ++++++---- .../mqtt/MqttPendingUnsubscription.java | 85 +++++-- .../org/thingsboard/mqtt/MqttPingHandler.java | 17 +- .../thingsboard/mqtt/PendingOperation.java | 4 +- .../mqtt/RetransmissionHandler.java | 101 +++++++-- .../org/thingsboard/mqtt/MqttClientTest.java | 210 ++++++++++++++++++ .../thingsboard/mqtt/MqttPingHandlerTest.java | 63 ------ .../org/thingsboard/mqtt/MqttTestProxy.java | 202 +++++++++++++++++ .../mqtt/integration/MqttIntegrationTest.java | 151 ------------- .../mqtt/integration/server/MqttServer.java | 84 ------- .../server/MqttTransportHandler.java | 141 ------------ .../test/resources/junit-platform.properties | 3 - pom.xml | 6 + .../rule/engine/api/MqttClientSettings.java | 26 +++ .../rule/engine/api/TbContext.java | 5 + .../rule/engine/mqtt/TbMqttNode.java | 8 + .../rule/engine/mqtt/TbMqttNodeTest.java | 18 ++ 33 files changed, 1100 insertions(+), 623 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientRetransmissionSettingsComponent.java create mode 100644 application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientSettingsComponent.java create mode 100644 netty-mqtt/src/main/java/org/thingsboard/mqtt/MaxRetransmissionsReachedException.java create mode 100644 netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttClientTest.java delete mode 100644 netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttPingHandlerTest.java create mode 100644 netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttTestProxy.java delete mode 100644 netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/MqttIntegrationTest.java delete mode 100644 netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/server/MqttServer.java delete mode 100644 netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/server/MqttTransportHandler.java delete mode 100644 netty-mqtt/src/test/resources/junit-platform.properties create mode 100644 rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/MqttClientSettings.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 1ed919e922..78819ab246 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -30,9 +30,10 @@ import org.springframework.context.annotation.Lazy; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.api.DeviceStateManager; import org.thingsboard.rule.engine.api.MailService; +import org.thingsboard.rule.engine.api.MqttClientSettings; import org.thingsboard.rule.engine.api.NotificationCenter; -import org.thingsboard.rule.engine.api.DeviceStateManager; import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.rule.engine.api.notification.SlackService; import org.thingsboard.rule.engine.api.sms.SmsSenderFactory; @@ -639,6 +640,10 @@ public class ActorSystemContext { @Getter private long cfCalculationResultTimeout; + @Autowired + @Getter + private MqttClientSettings mqttClientSettings; + @Getter @Setter private TbActorSystem actorSystem; diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 033e10ca9a..3fb28aee38 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -23,14 +23,15 @@ import org.bouncycastle.util.Arrays; import org.thingsboard.common.util.DebugModeUtil; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ListeningExecutor; +import org.thingsboard.rule.engine.api.DeviceStateManager; import org.thingsboard.rule.engine.api.MailService; +import org.thingsboard.rule.engine.api.MqttClientSettings; import org.thingsboard.rule.engine.api.NotificationCenter; import org.thingsboard.rule.engine.api.RuleEngineAlarmService; import org.thingsboard.rule.engine.api.RuleEngineApiUsageStateService; import org.thingsboard.rule.engine.api.RuleEngineAssetProfileCache; import org.thingsboard.rule.engine.api.RuleEngineCalculatedFieldQueueService; import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache; -import org.thingsboard.rule.engine.api.DeviceStateManager; import org.thingsboard.rule.engine.api.RuleEngineRpcService; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.ScriptEngine; @@ -1010,13 +1011,17 @@ public class DefaultTbContext implements TbContext { return mainCtx.getAuditLogService(); } + @Override + public MqttClientSettings getMqttClientSettings() { + return mainCtx.getMqttClientSettings(); + } + private TbMsgMetaData getActionMetaData(RuleNodeId ruleNodeId) { TbMsgMetaData metaData = new TbMsgMetaData(); metaData.putValue("ruleNodeId", ruleNodeId.toString()); return metaData; } - @Override public void schedule(Runnable runnable, long delay, TimeUnit timeUnit) { mainCtx.getScheduler().schedule(runnable, delay, timeUnit); diff --git a/application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientRetransmissionSettingsComponent.java b/application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientRetransmissionSettingsComponent.java new file mode 100644 index 0000000000..33e9358d2b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientRetransmissionSettingsComponent.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.config.mqtt; + +import jakarta.validation.constraints.PositiveOrZero; +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; +import org.springframework.validation.annotation.Validated; + +@Data +@Validated +@Configuration +@ConfigurationProperties(prefix = "mqtt.client.retransmission") +public class MqttClientRetransmissionSettingsComponent { + + @PositiveOrZero + private int maxAttempts; + @PositiveOrZero + private long initialDelayMillis; + @PositiveOrZero + private double jitterFactor; + +} diff --git a/application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientSettingsComponent.java b/application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientSettingsComponent.java new file mode 100644 index 0000000000..25df212925 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/config/mqtt/MqttClientSettingsComponent.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.config.mqtt; + +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; +import lombok.ToString; +import org.springframework.context.annotation.Configuration; +import org.thingsboard.rule.engine.api.MqttClientSettings; + +@ToString +@EqualsAndHashCode +@Configuration +@RequiredArgsConstructor +public class MqttClientSettingsComponent implements MqttClientSettings { + + private final MqttClientRetransmissionSettingsComponent retransmissionSettingsComponent; + + @Override + public int getRetransmissionMaxAttempts() { + return retransmissionSettingsComponent.getMaxAttempts(); + } + + @Override + public long getRetransmissionInitialDelayMillis() { + return retransmissionSettingsComponent.getInitialDelayMillis(); + } + + @Override + public double getRetransmissionJitterFactor() { + return retransmissionSettingsComponent.getJitterFactor(); + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 721d23b700..d7a55e496e 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1920,3 +1920,27 @@ mobileApp: googlePlayLink: "${TB_MOBILE_APP_GOOGLE_PLAY_LINK:https://play.google.com/store/apps/details?id=org.thingsboard.demo.app}" # Link to App Store for Thingsboard Live mobile application appStoreLink: "${TB_MOBILE_APP_APP_STORE_LINK:https://apps.apple.com/us/app/thingsboard-live/id1594355695}" + +mqtt: + # MQTT client configuration parameters + client: + # Parameters that control the retransmission mechanism. + # This mechanism only applies to the handling of MQTT Publish, Subscribe, Unsubscribe and Pubrel messages. + # With the updated default settings: + # - After sending the message, wait approximately 5000 ms (± jitter) for the 1st attempt. + # - The 2nd attempt will occur after roughly 5000 * 2 = 10,000 ms (± jitter). + # - The 3rd attempt will occur after roughly 5000 * 4 = 20,000 ms (± jitter). + # - The 4th "attempt" will not actually perform a retransmission. + # Instead, the system will detect that the maximum number of attempts has been reached and drop the pending message. + retransmission: + # Maximum number of retransmission attempts allowed. + # If the attempt count exceeds this value, retransmissions will stop and the pending message will be dropped. + max_attempts: "${TB_MQTT_CLIENT_RETRANSMISSION_MAX_ATTEMPTS:3}" + # Base delay (in milliseconds) before the first retransmission attempt, measured from the moment the message is sent. + # Subsequent delays are calculated using exponential backoff. + # This base delay is also used as the reference value for applying jitter. + initial_delay_millis: "${TB_MQTT_CLIENT_RETRANSMISSION_INITIAL_DELAY_MILLIS:5000}" + # Jitter factor applied to the calculated retransmission delay. + # The actual delay is randomized within a range defined by multiplying the base delay by a factor between (1 - jitter_factor) and (1 + jitter_factor). + # For example, a jitter_factor of 0.15 means the actual delay may vary by up to ±15% of the base delay. + jitter_factor: "${TB_MQTT_CLIENT_RETRANSMISSION_JITTER_FACTOR:0.15}" diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 65def6a964..9b9bb31dc1 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -55,8 +55,8 @@ public class ContainerTestSuite { private static final String TB_JS_EXECUTOR_LOG_REGEXP = ".*template started.*"; private static final Duration CONTAINER_STARTUP_TIMEOUT = Duration.ofSeconds(400); - private DockerComposeContainer testContainer; - private ThingsBoardDbInstaller installTb; + private DockerComposeContainer testContainer; + private ThingsBoardDbInstaller installTb; private boolean isActive; private static ContainerTestSuite containerTestSuite; @@ -194,7 +194,7 @@ public class ContainerTestSuite { setActive(true); } catch (Exception e) { log.error("Failed to create test container", e); - fail("Failed to create test container"); + fail("Failed to create test container", e); } } @@ -263,7 +263,7 @@ public class ContainerTestSuite { log.info("Trying to delete temp dir {}", targetDir); FileUtils.deleteDirectory(new File(targetDir)); } catch (IOException e) { - log.error("Can't delete temp directory " + targetDir, e); + log.error("Can't delete temp directory {}", targetDir, e); } } @@ -286,8 +286,8 @@ public class ContainerTestSuite { FileUtils.writeStringToFile(file, outputContent, StandardCharsets.UTF_8); assertThat(FileUtils.readFileToString(file, StandardCharsets.UTF_8), is(outputContent)); } catch (IOException e) { - log.error("failed to update file " + sourceFilename, e); - fail("failed to update file"); + log.error("failed to update file {}", sourceFilename, e); + fail("failed to update file", e); } } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java index ebdfb4e3c9..dacfea9b10 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java @@ -42,7 +42,6 @@ import org.thingsboard.mqtt.MqttClient; import org.thingsboard.mqtt.MqttClientCallback; import org.thingsboard.mqtt.MqttClientConfig; import org.thingsboard.mqtt.MqttHandler; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileProvisionType; @@ -82,7 +81,6 @@ import java.util.concurrent.TimeoutException; import static org.assertj.core.api.Assertions.assertThat; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.fail; -import static org.thingsboard.server.common.data.DataConstants.DEVICE; import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE; import static org.thingsboard.server.msa.prototypes.DevicePrototypes.defaultDevicePrototype; @@ -301,7 +299,7 @@ public class MqttClientTest extends AbstractContainerTest { assertThat(Objects.requireNonNull(requestFromServer).getMessage()).isEqualTo("{\"method\":\"getValue\",\"params\":true}"); - Integer requestId = Integer.valueOf(Objects.requireNonNull(requestFromServer).getTopic().substring("v1/devices/me/rpc/request/".length())); + int requestId = Integer.parseInt(Objects.requireNonNull(requestFromServer).getTopic().substring("v1/devices/me/rpc/request/".length())); JsonObject clientResponse = new JsonObject(); clientResponse.addProperty("response", "someResponse"); // Send a response to the server's RPC request @@ -340,7 +338,7 @@ public class MqttClientTest extends AbstractContainerTest { assertThat(Objects.requireNonNull(requestFromServer).getMessage()).isEqualTo("{\"method\":\"getValue\",\"params\":true}"); - Integer requestId = Integer.valueOf(Objects.requireNonNull(requestFromServer).getTopic().substring("v1/devices/me/rpc/request/".length())); + int requestId = Integer.parseInt(Objects.requireNonNull(requestFromServer).getTopic().substring("v1/devices/me/rpc/request/".length())); JsonObject clientResponse = new JsonObject(); clientResponse.addProperty("response", "someResponse"); // Send a response to the server's RPC request @@ -520,13 +518,13 @@ public class MqttClientTest extends AbstractContainerTest { mqttClient.on("/provision/response", listener, MqttQoS.AT_LEAST_ONCE).get(3 * timeoutMultiplier, TimeUnit.SECONDS); TimeUnit.SECONDS.sleep(2 * timeoutMultiplier); assertThat(subAckResult[0]).isNotNull(); - assertThat(MqttReasonCodes.SubAck.GRANTED_QOS_1.equals(subAckResult[0])); + assertThat(MqttReasonCodes.SubAck.GRANTED_QOS_1).isEqualTo(subAckResult[0]); subAckResult[0] = null; mqttClient.on("v1/devices/me/attributes", listener, MqttQoS.AT_LEAST_ONCE).get(3 * timeoutMultiplier, TimeUnit.SECONDS); TimeUnit.SECONDS.sleep(2 * timeoutMultiplier); assertThat(subAckResult[0]).isNotNull(); - assertThat(MqttReasonCodes.SubAck.TOPIC_FILTER_INVALID.equals(subAckResult[0])); + assertThat(MqttReasonCodes.SubAck.TOPIC_FILTER_INVALID).isEqualTo(subAckResult[0]); testRestClient.deleteDeviceIfExists(device.getId()); updateDeviceProfileWithProvisioningStrategy(deviceProfile, DeviceProfileProvisionType.DISABLED); @@ -596,7 +594,7 @@ public class MqttClientTest extends AbstractContainerTest { .await() .alias("Check device disconnect.") .atMost(TIMEOUT*timeoutMultiplier, TimeUnit.SECONDS) - .until(() -> returnCodeByteValue.size() > 0); + .until(() -> !returnCodeByteValue.isEmpty()); assertThat(returnCodeByteValueSecondClient).isEmpty(); assertThat(returnCodeByteValue).isNotEmpty(); @@ -663,7 +661,7 @@ public class MqttClientTest extends AbstractContainerTest { .stream() .filter(RuleChain::isRoot) .findFirst(); - if (!defaultRuleChain.isPresent()) { + if (defaultRuleChain.isEmpty()) { fail("Root rule chain wasn't found"); } return defaultRuleChain.get().getId(); @@ -717,6 +715,7 @@ public class MqttClientTest extends AbstractContainerTest { clientConfig.setClientId("MQTT client from test"); clientConfig.setUsername(username); clientConfig.setProtocolVersion(mqttVersion); + clientConfig.setRetransmissionConfig(new MqttClientConfig.RetransmissionConfig(5, 5000L, 0.1d)); // same as defaults in thingsboard.yml as of time of this writing MqttClient mqttClient = MqttClient.create(clientConfig, listener, handlerExecutor); if (connect) { mqttClient.connect(TRANSPORT_HOST, TRANSPORT_PORT).get(); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java index cc587fbbd5..32e8498f45 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java @@ -39,7 +39,6 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.mqtt.MqttClient; import org.thingsboard.mqtt.MqttClientConfig; import org.thingsboard.mqtt.MqttHandler; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.DeviceId; @@ -65,7 +64,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; -import static org.thingsboard.server.common.data.DataConstants.DEVICE; import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE; import static org.thingsboard.server.msa.prototypes.DevicePrototypes.defaultGatewayPrototype; @@ -76,7 +74,6 @@ public class MqttGatewayClientTest extends AbstractContainerTest { private MqttClient mqttClient; private Device createdDevice; private MqttMessageListener listener; - private JsonParser jsonParser = new JsonParser(); AbstractListeningExecutor handlerExecutor; @@ -100,7 +97,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest { } @AfterMethod - public void removeGateway() { + public void removeGateway() { testRestClient.deleteDeviceIfExists(this.gatewayDevice.getId()); testRestClient.deleteDeviceIfExists(this.createdDevice.getId()); this.listener = null; @@ -197,7 +194,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest { mqttClient.publish("v1/gateway/attributes/request", Unpooled.wrappedBuffer(requestData.toString().getBytes())).get(); event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); - JsonObject responseData = jsonParser.parse(Objects.requireNonNull(event).getMessage()).getAsJsonObject(); + JsonObject responseData = JsonParser.parseString(Objects.requireNonNull(event).getMessage()).getAsJsonObject(); assertThat(responseData.has("value")).isTrue(); assertThat(responseData.get("value").getAsString()).isEqualTo(sharedAttributes.get("attr1").getAsString()); @@ -213,7 +210,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest { mqttClient.on("v1/gateway/attributes/response", listener, MqttQoS.AT_LEAST_ONCE).get(); mqttClient.publish("v1/gateway/attributes/request", Unpooled.wrappedBuffer(requestData.toString().getBytes())).get(); event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); - responseData = jsonParser.parse(Objects.requireNonNull(event).getMessage()).getAsJsonObject(); + responseData = JsonParser.parseString(Objects.requireNonNull(event).getMessage()).getAsJsonObject(); assertThat(responseData.has("values")).isTrue(); assertThat(responseData.get("values").getAsJsonObject().get("attr1").getAsString()).isEqualTo(sharedAttributes.get("attr1").getAsString()); @@ -231,7 +228,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest { mqttClient.on("v1/gateway/attributes/response", listener, MqttQoS.AT_LEAST_ONCE).get(); mqttClient.publish("v1/gateway/attributes/request", Unpooled.wrappedBuffer(requestData.toString().getBytes())).get(); event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); - responseData = jsonParser.parse(Objects.requireNonNull(event).getMessage()).getAsJsonObject(); + responseData = JsonParser.parseString(Objects.requireNonNull(event).getMessage()).getAsJsonObject(); assertThat(responseData.has("values")).isTrue(); assertThat(responseData.get("values").getAsJsonObject().get("attr1").getAsString()).isEqualTo(sharedAttributes.get("attr1").getAsString()); @@ -390,7 +387,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest { mqttClient.publish("v1/gateway/attributes/request", Unpooled.wrappedBuffer(gatewayAttributesRequest.toString().getBytes())).get(); MqttEvent clientAttributeEvent = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); assertThat(clientAttributeEvent).isNotNull(); - JsonObject responseMessage = new JsonParser().parse(Objects.requireNonNull(clientAttributeEvent).getMessage()).getAsJsonObject(); + JsonObject responseMessage = JsonParser.parseString(Objects.requireNonNull(clientAttributeEvent).getMessage()).getAsJsonObject(); assertThat(responseMessage.get("id").getAsInt()).isEqualTo(messageId); assertThat(responseMessage.get("device").getAsString()).isEqualTo(createdDevice.getName()); @@ -427,6 +424,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest { clientConfig.setOwnerId(getOwnerId()); clientConfig.setClientId("MQTT client from test"); clientConfig.setUsername(deviceCredentials.getCredentialsId()); + clientConfig.setRetransmissionConfig(new MqttClientConfig.RetransmissionConfig(3, 5000L, 0.1d)); // same as defaults in thingsboard.yml as of time of this writing MqttClient mqttClient = MqttClient.create(clientConfig, listener, handlerExecutor); mqttClient.connect("localhost", 1883).get(); return mqttClient; diff --git a/netty-mqtt/pom.xml b/netty-mqtt/pom.xml index b5fd83a54f..f9aa80c78e 100644 --- a/netty-mqtt/pom.xml +++ b/netty-mqtt/pom.xml @@ -87,6 +87,26 @@ awaitility test + + org.testcontainers + testcontainers + test + + + org.testcontainers + junit-jupiter + test + + + software.xdev + testcontainers-junit4-mock + test + + + org.testcontainers + hivemq + test + diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MaxRetransmissionsReachedException.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MaxRetransmissionsReachedException.java new file mode 100644 index 0000000000..3d483dd541 --- /dev/null +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MaxRetransmissionsReachedException.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt; + +public class MaxRetransmissionsReachedException extends RuntimeException { + + public MaxRetransmissionsReachedException(String message) { + super(message); + } + +} diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttChannelHandler.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttChannelHandler.java index ad976c848a..9686a2b1d7 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttChannelHandler.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttChannelHandler.java @@ -57,7 +57,7 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler } @Override - protected void channelRead0(ChannelHandlerContext ctx, MqttMessage msg) throws Exception { + protected void channelRead0(ChannelHandlerContext ctx, MqttMessage msg) { if (msg.decoderResult().isSuccess()) { switch (msg.fixedHeader().messageType()) { case CONNACK: @@ -120,6 +120,7 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler this.client.getClientConfig().getUsername(), this.client.getClientConfig().getPassword() != null ? this.client.getClientConfig().getPassword().getBytes(CharsetUtil.UTF_8) : null ); + log.debug("{} Sending CONNECT", client.getClientConfig().getOwnerId()); ctx.channel().writeAndFlush(new MqttConnectMessage(fixedHeader, variableHeader, payload)); } @@ -173,6 +174,7 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler } private void handleConack(Channel channel, MqttConnAckMessage message) { + log.debug("{} Handling CONNACK", client.getClientConfig().getOwnerId()); switch (message.variableHeader().connectReturnCode()) { case CONNECTION_ACCEPTED: this.connectFuture.setSuccess(new MqttConnectResult(true, MqttConnectReturnCode.CONNECTION_ACCEPTED, channel.closeFuture())); @@ -219,9 +221,9 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler } pendingSubscription.onSubackReceived(); for (MqttPendingSubscription.MqttPendingHandler handler : pendingSubscription.getHandlers()) { - MqttSubscription subscription = new MqttSubscription(pendingSubscription.getTopic(), handler.getHandler(), handler.isOnce()); + MqttSubscription subscription = new MqttSubscription(pendingSubscription.getTopic(), handler.handler(), handler.once()); this.client.getSubscriptions().put(pendingSubscription.getTopic(), subscription); - this.client.getHandlerToSubscription().put(handler.getHandler(), subscription); + this.client.getHandlerToSubscription().put(handler.handler(), subscription); } this.client.getPendingSubscribeTopics().remove(pendingSubscription.getTopic()); @@ -282,17 +284,16 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler } private void handlePuback(MqttPubAckMessage message) { - MqttPendingPublish pendingPublish = this.client.getPendingPublishes().get(message.variableHeader().messageId()); - if (pendingPublish == null) { - return; - } - pendingPublish.getFuture().setSuccess(null); - pendingPublish.onPubackReceived(); - this.client.getPendingPublishes().remove(message.variableHeader().messageId()); - pendingPublish.getPayload().release(); - if (this.client.getCallback() != null) { - this.client.getCallback().onPubAck(message); - } + log.trace("{} Handling PUBACK", client.getClientConfig().getOwnerId()); + client.getPendingPublishes().computeIfPresent(message.variableHeader().messageId(), (__, pendingPublish) -> { + pendingPublish.getFuture().setSuccess(null); + pendingPublish.onPubackReceived(); + pendingPublish.getPayload().release(); + if (client.getCallback() != null) { + client.getCallback().onPubAck(message); + } + return null; + }); } private void handlePubrec(Channel channel, MqttMessage message) { @@ -335,6 +336,7 @@ final class MqttChannelHandler extends SimpleChannelInboundHandler } private void handleDisconnect(MqttMessage message) { + log.debug("{} Handling DISCONNECT", client.getClientConfig().getOwnerId()); if (this.client.getCallback() != null) { this.client.getCallback().onDisconnect(message); } diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClient.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClient.java index db0459e08a..4d845320e8 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClient.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClient.java @@ -184,7 +184,7 @@ public interface MqttClient { * @param config The config object to use while looking for settings * @param defaultHandler The handler for incoming messages that do not match any topic subscriptions */ - static MqttClient create(MqttClientConfig config, MqttHandler defaultHandler, ListeningExecutor handlerExecutor){ + static MqttClient create(MqttClientConfig config, MqttHandler defaultHandler, ListeningExecutor handlerExecutor) { return new MqttClientImpl(config, defaultHandler, handlerExecutor); } diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientConfig.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientConfig.java index 41df077d71..24feb3e58e 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientConfig.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientConfig.java @@ -47,6 +47,26 @@ public final class MqttClientConfig { private long reconnectDelay = 1L; private int maxBytesInMessage = 8092; + @Getter + @Setter + private RetransmissionConfig retransmissionConfig; + + public record RetransmissionConfig(int maxAttempts, long initialDelayMillis, double jitterFactor) { + + public RetransmissionConfig { + if (maxAttempts < 0) { + throw new IllegalArgumentException("Max retransmission attempts (maxAttempts) must be zero or greater, but was " + maxAttempts); + } + if (initialDelayMillis < 0) { + throw new IllegalArgumentException("Initial retransmission delay (initialDelayMillis) must be zero or greater, but was " + initialDelayMillis); + } + if (jitterFactor < 0) { + throw new IllegalArgumentException("Jitter factor (jitterFactor) must be zero or greater, but was " + jitterFactor); + } + } + + } + public MqttClientConfig() { this(null); } diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java index 47eae565dc..ee07752db3 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttClientImpl.java @@ -17,6 +17,7 @@ package org.thingsboard.mqtt; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; import io.netty.bootstrap.Bootstrap; import io.netty.buffer.ByteBuf; import io.netty.channel.Channel; @@ -384,8 +385,33 @@ final class MqttClientImpl implements MqttClient { MqttFixedHeader fixedHeader = new MqttFixedHeader(MqttMessageType.PUBLISH, false, qos, retain, 0); MqttPublishVariableHeader variableHeader = new MqttPublishVariableHeader(topic, getNewMessageId().messageId()); MqttPublishMessage message = new MqttPublishMessage(fixedHeader, variableHeader, payload); - MqttPendingPublish pendingPublish = new MqttPendingPublish(variableHeader.packetId(), future, - payload.retain(), message, qos, () -> !pendingPublishes.containsKey(variableHeader.packetId())); + + final var pendingPublish = MqttPendingPublish.builder() + .messageId(variableHeader.packetId()) + .future(future) + .payload(payload.retain()) + .message(message) + .qos(qos) + .ownerId(clientConfig.getOwnerId()) + .retransmissionConfig(clientConfig.getRetransmissionConfig()) + .pendingOperation(new PendingOperation() { + @Override + public boolean isCancelled() { + return !pendingPublishes.containsKey(variableHeader.packetId()); + } + + @Override + public void onMaxRetransmissionAttemptsReached() { + pendingPublishes.computeIfPresent(variableHeader.packetId(), (__, pendingPublish) -> { + var message = "Unable to deliver publish message due to max retransmission attempts (%s) being reached for client '%s' on topic '%s' (message ID: %d)" + .formatted(clientConfig.getRetransmissionConfig().maxAttempts(), clientConfig.getClientId(), topic, variableHeader.packetId()); + pendingPublish.getFuture().tryFailure(new MaxRetransmissionsReachedException(message)); + pendingPublish.getPayload().release(); + return null; + }); + } + }).build(); + this.pendingPublishes.put(pendingPublish.getMessageId(), pendingPublish); ChannelFuture channelFuture = this.sendAndFlushPacket(message); @@ -499,9 +525,30 @@ final class MqttClientImpl implements MqttClient { MqttSubscribePayload payload = new MqttSubscribePayload(Collections.singletonList(subscription)); MqttSubscribeMessage message = new MqttSubscribeMessage(fixedHeader, variableHeader, payload); - final MqttPendingSubscription pendingSubscription = new MqttPendingSubscription(future, topic, message, - () -> !pendingSubscriptions.containsKey(variableHeader.messageId())); - pendingSubscription.addHandler(handler, once); + final var pendingSubscription = MqttPendingSubscription.builder() + .future(future) + .topic(topic) + .handlers(Sets.newHashSet(new MqttPendingSubscription.MqttPendingHandler(handler, once))) + .subscribeMessage(message) + .ownerId(clientConfig.getOwnerId()) + .retransmissionConfig(clientConfig.getRetransmissionConfig()) + .pendingOperation(new PendingOperation() { + @Override + public boolean isCancelled() { + return !pendingSubscriptions.containsKey(variableHeader.messageId()); + } + + @Override + public void onMaxRetransmissionAttemptsReached() { + pendingSubscriptions.computeIfPresent(variableHeader.messageId(), (__, pendingSubscription) -> { + var message = "Unable to deliver subscribe message due to max retransmission attempts (%s) being reached for client '%s' on topic '%s' (message ID: %d)" + .formatted(clientConfig.getRetransmissionConfig().maxAttempts(), clientConfig.getClientId(), topic, variableHeader.messageId()); + pendingSubscription.getFuture().tryFailure(new MaxRetransmissionsReachedException(message)); + return null; + }); + } + }).build(); + this.pendingSubscriptions.put(variableHeader.messageId(), pendingSubscription); this.pendingSubscribeTopics.add(topic); pendingSubscription.setSent(this.sendAndFlushPacket(message) != null); //If not sent, we will send it when the connection is opened @@ -518,8 +565,29 @@ final class MqttClientImpl implements MqttClient { MqttUnsubscribePayload payload = new MqttUnsubscribePayload(Collections.singletonList(topic)); MqttUnsubscribeMessage message = new MqttUnsubscribeMessage(fixedHeader, variableHeader, payload); - MqttPendingUnsubscription pendingUnsubscription = new MqttPendingUnsubscription(promise, topic, message, - () -> !pendingServerUnsubscribes.containsKey(variableHeader.messageId())); + final var pendingUnsubscription = MqttPendingUnsubscription.builder() + .future(promise) + .topic(topic) + .unsubscribeMessage(message) + .ownerId(clientConfig.getOwnerId()) + .retransmissionConfig(clientConfig.getRetransmissionConfig()) + .pendingOperation(new PendingOperation() { + @Override + public boolean isCancelled() { + return !pendingServerUnsubscribes.containsKey(variableHeader.messageId()); + } + + @Override + public void onMaxRetransmissionAttemptsReached() { + pendingServerUnsubscribes.computeIfPresent(variableHeader.messageId(), (__, pendingUnsubscription) -> { + var message = "Unable to deliver unsubscribe message due to max retransmission attempts (%s) being reached for client '%s' on topic '%s' (message ID: %d)" + .formatted(clientConfig.getRetransmissionConfig().maxAttempts(), clientConfig.getClientId(), topic, variableHeader.messageId()); + pendingUnsubscription.getFuture().tryFailure(new MaxRetransmissionsReachedException(message)); + return null; + }); + } + }).build(); + this.pendingServerUnsubscribes.put(variableHeader.messageId(), pendingUnsubscription); pendingUnsubscription.startRetransmissionTimer(this.eventLoop.next(), this::sendAndFlushPacket); diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttConnectResult.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttConnectResult.java index 911bc1d395..67757d2a7a 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttConnectResult.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttConnectResult.java @@ -17,7 +17,9 @@ package org.thingsboard.mqtt; import io.netty.channel.ChannelFuture; import io.netty.handler.codec.mqtt.MqttConnectReturnCode; +import lombok.ToString; +@ToString @SuppressWarnings({"WeakerAccess", "unused"}) public final class MqttConnectResult { @@ -42,4 +44,5 @@ public final class MqttConnectResult { public ChannelFuture getCloseFuture() { return closeFuture; } + } diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingPublish.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingPublish.java index e8c3ef35f7..1846bdb12b 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingPublish.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingPublish.java @@ -21,9 +21,13 @@ import io.netty.handler.codec.mqtt.MqttMessage; import io.netty.handler.codec.mqtt.MqttPublishMessage; import io.netty.handler.codec.mqtt.MqttQoS; import io.netty.util.concurrent.Promise; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.Setter; import java.util.function.Consumer; +@Getter(AccessLevel.PACKAGE) final class MqttPendingPublish { private final int messageId; @@ -32,80 +36,126 @@ final class MqttPendingPublish { private final MqttPublishMessage message; private final MqttQoS qos; + @Getter(AccessLevel.NONE) private final RetransmissionHandler publishRetransmissionHandler; + @Getter(AccessLevel.NONE) private final RetransmissionHandler pubrelRetransmissionHandler; + @Setter(AccessLevel.PACKAGE) private boolean sent = false; - MqttPendingPublish(int messageId, Promise future, ByteBuf payload, MqttPublishMessage message, MqttQoS qos, PendingOperation operation) { + private MqttPendingPublish( + int messageId, + Promise future, + ByteBuf payload, + MqttPublishMessage message, + MqttQoS qos, + String ownerId, + MqttClientConfig.RetransmissionConfig retransmissionConfig, + PendingOperation pendingOperation + ) { this.messageId = messageId; this.future = future; this.payload = payload; this.message = message; this.qos = qos; - this.publishRetransmissionHandler = new RetransmissionHandler<>(operation); - this.publishRetransmissionHandler.setOriginalMessage(message); - this.pubrelRetransmissionHandler = new RetransmissionHandler<>(operation); - } - - int getMessageId() { - return messageId; - } - - Promise getFuture() { - return future; - } - - ByteBuf getPayload() { - return payload; - } - - boolean isSent() { - return sent; - } - - void setSent(boolean sent) { - this.sent = sent; - } - - MqttPublishMessage getMessage() { - return message; - } - - MqttQoS getQos() { - return qos; + publishRetransmissionHandler = new RetransmissionHandler<>(retransmissionConfig, pendingOperation, ownerId); + publishRetransmissionHandler.setOriginalMessage(message); + pubrelRetransmissionHandler = new RetransmissionHandler<>(retransmissionConfig, pendingOperation, ownerId); } void startPublishRetransmissionTimer(EventLoop eventLoop, Consumer sendPacket) { - this.publishRetransmissionHandler.setHandle(((fixedHeader, originalMessage) -> - sendPacket.accept(new MqttPublishMessage(fixedHeader, originalMessage.variableHeader(), this.payload.retain())))); - this.publishRetransmissionHandler.start(eventLoop); + publishRetransmissionHandler.setHandler(((fixedHeader, originalMessage) -> + sendPacket.accept(new MqttPublishMessage(fixedHeader, originalMessage.variableHeader(), payload.retain())))); + publishRetransmissionHandler.start(eventLoop); } void onPubackReceived() { - this.publishRetransmissionHandler.stop(); + publishRetransmissionHandler.stop(); } void setPubrelMessage(MqttMessage pubrelMessage) { - this.pubrelRetransmissionHandler.setOriginalMessage(pubrelMessage); + pubrelRetransmissionHandler.setOriginalMessage(pubrelMessage); } void startPubrelRetransmissionTimer(EventLoop eventLoop, Consumer sendPacket) { - this.pubrelRetransmissionHandler.setHandle((fixedHeader, originalMessage) -> + pubrelRetransmissionHandler.setHandler((fixedHeader, originalMessage) -> sendPacket.accept(new MqttMessage(fixedHeader, originalMessage.variableHeader()))); - this.pubrelRetransmissionHandler.start(eventLoop); + pubrelRetransmissionHandler.start(eventLoop); } void onPubcompReceived() { - this.pubrelRetransmissionHandler.stop(); + pubrelRetransmissionHandler.stop(); } void onChannelClosed() { - this.publishRetransmissionHandler.stop(); - this.pubrelRetransmissionHandler.stop(); + publishRetransmissionHandler.stop(); + pubrelRetransmissionHandler.stop(); if (payload != null) { payload.release(); } } + + static Builder builder() { + return new Builder(); + } + + static class Builder { + + private int messageId; + private Promise future; + private ByteBuf payload; + private MqttPublishMessage message; + private MqttQoS qos; + private String ownerId; + private MqttClientConfig.RetransmissionConfig retransmissionConfig; + private PendingOperation pendingOperation; + + Builder messageId(int messageId) { + this.messageId = messageId; + return this; + } + + Builder future(Promise future) { + this.future = future; + return this; + } + + Builder payload(ByteBuf payload) { + this.payload = payload; + return this; + } + + Builder message(MqttPublishMessage message) { + this.message = message; + return this; + } + + Builder qos(MqttQoS qos) { + this.qos = qos; + return this; + } + + Builder ownerId(String ownerId) { + this.ownerId = ownerId; + return this; + } + + Builder retransmissionConfig(MqttClientConfig.RetransmissionConfig retransmissionConfig) { + this.retransmissionConfig = retransmissionConfig; + return this; + } + + Builder pendingOperation(PendingOperation pendingOperation) { + this.pendingOperation = pendingOperation; + return this; + } + + MqttPendingPublish build() { + return new MqttPendingPublish(messageId, future, payload, message, qos, ownerId, retransmissionConfig, pendingOperation); + } + + } + } diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingSubscription.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingSubscription.java index af5d53a06c..7b2ba613cb 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingSubscription.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingSubscription.java @@ -18,90 +18,123 @@ package org.thingsboard.mqtt; import io.netty.channel.EventLoop; import io.netty.handler.codec.mqtt.MqttSubscribeMessage; import io.netty.util.concurrent.Promise; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.Setter; import java.util.HashSet; import java.util.Set; import java.util.function.Consumer; +import static java.util.Objects.requireNonNullElseGet; + +@Getter(AccessLevel.PACKAGE) final class MqttPendingSubscription { private final Promise future; private final String topic; - private final Set handlers = new HashSet<>(); + private final Set handlers; private final MqttSubscribeMessage subscribeMessage; + @Getter(AccessLevel.NONE) private final RetransmissionHandler retransmissionHandler; + @Setter(AccessLevel.PACKAGE) private boolean sent = false; - MqttPendingSubscription(Promise future, String topic, MqttSubscribeMessage message, PendingOperation operation) { + private MqttPendingSubscription( + Promise future, + String topic, + Set handlers, + MqttSubscribeMessage subscribeMessage, + String ownerId, + MqttClientConfig.RetransmissionConfig retransmissionConfig, + PendingOperation operation + ) { this.future = future; this.topic = topic; - this.subscribeMessage = message; + this.handlers = requireNonNullElseGet(handlers, HashSet::new); + this.subscribeMessage = subscribeMessage; - this.retransmissionHandler = new RetransmissionHandler<>(operation); - this.retransmissionHandler.setOriginalMessage(message); + retransmissionHandler = new RetransmissionHandler<>(retransmissionConfig, operation, ownerId); + retransmissionHandler.setOriginalMessage(subscribeMessage); } - Promise getFuture() { - return future; - } + record MqttPendingHandler(MqttHandler handler, boolean once) {} - String getTopic() { - return topic; + void addHandler(MqttHandler handler, boolean once) { + handlers.add(new MqttPendingHandler(handler, once)); } - boolean isSent() { - return sent; + void startRetransmitTimer(EventLoop eventLoop, Consumer sendPacket) { + if (sent) { // If the packet is sent, we can start the retransmission timer + retransmissionHandler.setHandler((fixedHeader, originalMessage) -> + sendPacket.accept(new MqttSubscribeMessage(fixedHeader, originalMessage.variableHeader(), originalMessage.payload()))); + retransmissionHandler.start(eventLoop); + } } - void setSent(boolean sent) { - this.sent = sent; + void onSubackReceived() { + retransmissionHandler.stop(); } - MqttSubscribeMessage getSubscribeMessage() { - return subscribeMessage; + void onChannelClosed() { + retransmissionHandler.stop(); } - void addHandler(MqttHandler handler, boolean once) { - this.handlers.add(new MqttPendingHandler(handler, once)); + static Builder builder() { + return new Builder(); } - Set getHandlers() { - return handlers; - } + static class Builder { - void startRetransmitTimer(EventLoop eventLoop, Consumer sendPacket) { - if (this.sent) { //If the packet is sent, we can start the retransmit timer - this.retransmissionHandler.setHandle((fixedHeader, originalMessage) -> - sendPacket.accept(new MqttSubscribeMessage(fixedHeader, originalMessage.variableHeader(), originalMessage.payload()))); - this.retransmissionHandler.start(eventLoop); + private Promise future; + private String topic; + private Set handlers; + private MqttSubscribeMessage subscribeMessage; + private String ownerId; + private PendingOperation pendingOperation; + private MqttClientConfig.RetransmissionConfig retransmissionConfig; + + Builder future(Promise future) { + this.future = future; + return this; } - } - void onSubackReceived() { - this.retransmissionHandler.stop(); - } + Builder topic(String topic) { + this.topic = topic; + return this; + } - final class MqttPendingHandler { - private final MqttHandler handler; - private final boolean once; + Builder handlers(Set handlers) { + this.handlers = handlers; + return this; + } - MqttPendingHandler(MqttHandler handler, boolean once) { - this.handler = handler; - this.once = once; + Builder subscribeMessage(MqttSubscribeMessage subscribeMessage) { + this.subscribeMessage = subscribeMessage; + return this; } - MqttHandler getHandler() { - return handler; + Builder ownerId(String ownerId) { + this.ownerId = ownerId; + return this; } - boolean isOnce() { - return once; + Builder retransmissionConfig(MqttClientConfig.RetransmissionConfig retransmissionConfig) { + this.retransmissionConfig = retransmissionConfig; + return this; + } + + Builder pendingOperation(PendingOperation pendingOperation) { + this.pendingOperation = pendingOperation; + return this; + } + + MqttPendingSubscription build() { + return new MqttPendingSubscription(future, topic, handlers, subscribeMessage, ownerId, retransmissionConfig, pendingOperation); } - } - void onChannelClosed() { - this.retransmissionHandler.stop(); } + } diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingUnsubscription.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingUnsubscription.java index 9cb3bd2f8d..8bc23292f8 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingUnsubscription.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPendingUnsubscription.java @@ -18,43 +18,96 @@ package org.thingsboard.mqtt; import io.netty.channel.EventLoop; import io.netty.handler.codec.mqtt.MqttUnsubscribeMessage; import io.netty.util.concurrent.Promise; +import lombok.AccessLevel; +import lombok.Getter; import java.util.function.Consumer; -final class MqttPendingUnsubscription{ +@Getter(AccessLevel.PACKAGE) +final class MqttPendingUnsubscription { private final Promise future; private final String topic; + @Getter(AccessLevel.NONE) private final RetransmissionHandler retransmissionHandler; - MqttPendingUnsubscription(Promise future, String topic, MqttUnsubscribeMessage unsubscribeMessage, PendingOperation operation) { + private MqttPendingUnsubscription( + Promise future, + String topic, + MqttUnsubscribeMessage unsubscribeMessage, + String ownerId, + MqttClientConfig.RetransmissionConfig retransmissionConfig, + PendingOperation operation + ) { this.future = future; this.topic = topic; - this.retransmissionHandler = new RetransmissionHandler<>(operation); - this.retransmissionHandler.setOriginalMessage(unsubscribeMessage); + retransmissionHandler = new RetransmissionHandler<>(retransmissionConfig, operation, ownerId); + retransmissionHandler.setOriginalMessage(unsubscribeMessage); } - Promise getFuture() { - return future; + void startRetransmissionTimer(EventLoop eventLoop, Consumer sendPacket) { + retransmissionHandler.setHandler((fixedHeader, originalMessage) -> + sendPacket.accept(new MqttUnsubscribeMessage(fixedHeader, originalMessage.variableHeader(), originalMessage.payload()))); + retransmissionHandler.start(eventLoop); } - String getTopic() { - return topic; + void onUnsubackReceived() { + retransmissionHandler.stop(); } - void startRetransmissionTimer(EventLoop eventLoop, Consumer sendPacket) { - this.retransmissionHandler.setHandle((fixedHeader, originalMessage) -> - sendPacket.accept(new MqttUnsubscribeMessage(fixedHeader, originalMessage.variableHeader(), originalMessage.payload()))); - this.retransmissionHandler.start(eventLoop); + void onChannelClosed() { + retransmissionHandler.stop(); } - void onUnsubackReceived(){ - this.retransmissionHandler.stop(); + static Builder builder() { + return new Builder(); } - void onChannelClosed(){ - this.retransmissionHandler.stop(); + static class Builder { + + private Promise future; + private String topic; + private MqttUnsubscribeMessage unsubscribeMessage; + private String ownerId; + private PendingOperation pendingOperation; + private MqttClientConfig.RetransmissionConfig retransmissionConfig; + + Builder future(Promise future) { + this.future = future; + return this; + } + + Builder topic(String topic) { + this.topic = topic; + return this; + } + + Builder unsubscribeMessage(MqttUnsubscribeMessage unsubscribeMessage) { + this.unsubscribeMessage = unsubscribeMessage; + return this; + } + + Builder ownerId(String ownerId) { + this.ownerId = ownerId; + return this; + } + + Builder retransmissionConfig(MqttClientConfig.RetransmissionConfig retransmissionConfig) { + this.retransmissionConfig = retransmissionConfig; + return this; + } + + Builder pendingOperation(PendingOperation pendingOperation) { + this.pendingOperation = pendingOperation; + return this; + } + + MqttPendingUnsubscription build() { + return new MqttPendingUnsubscription(future, topic, unsubscribeMessage, ownerId, retransmissionConfig, pendingOperation); + } + } + } diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPingHandler.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPingHandler.java index 70a4992d72..3fc2c6246e 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPingHandler.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/MqttPingHandler.java @@ -42,12 +42,11 @@ final class MqttPingHandler extends ChannelInboundHandlerAdapter { } @Override - public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { - if (!(msg instanceof MqttMessage)) { + public void channelRead(ChannelHandlerContext ctx, Object msg) { + if (!(msg instanceof MqttMessage message)) { ctx.fireChannelRead(msg); return; } - MqttMessage message = (MqttMessage) msg; if (message.fixedHeader().messageType() == MqttMessageType.PINGREQ) { this.handlePingReq(ctx.channel()); } else if (message.fixedHeader().messageType() == MqttMessageType.PINGRESP) { @@ -61,28 +60,29 @@ final class MqttPingHandler extends ChannelInboundHandlerAdapter { public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { super.userEventTriggered(ctx, evt); - if (evt instanceof IdleStateEvent) { - IdleStateEvent event = (IdleStateEvent) evt; + if (evt instanceof IdleStateEvent event) { switch (event.state()) { case READER_IDLE: log.debug("[{}] No reads were performed for specified period for channel {}", event.state(), ctx.channel().id()); - this.sendPingReq(ctx.channel()); + this.sendPingReq(ctx.channel(), event); break; case WRITER_IDLE: log.debug("[{}] No writes were performed for specified period for channel {}", event.state(), ctx.channel().id()); - this.sendPingReq(ctx.channel()); + this.sendPingReq(ctx.channel(), event); break; } } } - private void sendPingReq(Channel channel) { + private void sendPingReq(Channel channel, IdleStateEvent idleEvent) { log.trace("[{}] Sending ping request", channel.id()); MqttFixedHeader fixedHeader = new MqttFixedHeader(MqttMessageType.PINGREQ, false, MqttQoS.AT_MOST_ONCE, false, 0); channel.writeAndFlush(new MqttMessage(fixedHeader)); if (this.pingRespTimeout == null) { + log.trace("[{}] Scheduling disconnect due to {}", channel.id(), idleEvent); this.pingRespTimeout = channel.eventLoop().schedule(() -> { + log.trace("[{}] Sending disconnect due to {}", channel.id(), idleEvent); MqttFixedHeader fixedHeader2 = new MqttFixedHeader(MqttMessageType.DISCONNECT, false, MqttQoS.AT_MOST_ONCE, false, 0); channel.writeAndFlush(new MqttMessage(fixedHeader2)).addListener(ChannelFutureListener.CLOSE); //TODO: what do when the connection is closed ? @@ -99,6 +99,7 @@ final class MqttPingHandler extends ChannelInboundHandlerAdapter { private void handlePingResp(Channel channel) { log.trace("[{}] Handling ping response", channel.id()); if (this.pingRespTimeout != null && !this.pingRespTimeout.isCancelled() && !this.pingRespTimeout.isDone()) { + log.trace("[{}] Cancelling disconnect due to idle event because ping response was received", channel.id()); this.pingRespTimeout.cancel(true); this.pingRespTimeout = null; } diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/PendingOperation.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/PendingOperation.java index b859b216e6..07e472abb3 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/PendingOperation.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/PendingOperation.java @@ -17,6 +17,8 @@ package org.thingsboard.mqtt; public interface PendingOperation { - boolean isCanceled(); + boolean isCancelled(); + + void onMaxRetransmissionAttemptsReached(); } diff --git a/netty-mqtt/src/main/java/org/thingsboard/mqtt/RetransmissionHandler.java b/netty-mqtt/src/main/java/org/thingsboard/mqtt/RetransmissionHandler.java index b0d9ba9002..1778abc593 100644 --- a/netty-mqtt/src/main/java/org/thingsboard/mqtt/RetransmissionHandler.java +++ b/netty-mqtt/src/main/java/org/thingsboard/mqtt/RetransmissionHandler.java @@ -18,66 +18,119 @@ package org.thingsboard.mqtt; import io.netty.channel.EventLoop; import io.netty.handler.codec.mqtt.MqttFixedHeader; import io.netty.handler.codec.mqtt.MqttMessage; +import io.netty.handler.codec.mqtt.MqttMessageIdVariableHeader; import io.netty.handler.codec.mqtt.MqttMessageType; +import io.netty.handler.codec.mqtt.MqttPublishVariableHeader; import io.netty.handler.codec.mqtt.MqttQoS; import io.netty.util.concurrent.ScheduledFuture; import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; +@Slf4j @RequiredArgsConstructor final class RetransmissionHandler { - private volatile boolean stopped; + private final MqttClientConfig.RetransmissionConfig config; private final PendingOperation pendingOperation; + + private volatile boolean stopped; private ScheduledFuture timer; - private int timeout = 10; + private int attemptCount = 0; + + @Setter private BiConsumer handler; + + // the three fields below are used for logging only + private final String ownerId; + private String originalMessageId; + private long totalWaitingTimeMillis; + private T originalMessage; + void setOriginalMessage(T originalMessage) { + this.originalMessage = originalMessage; + var variableHeader = originalMessage.variableHeader(); + if (variableHeader instanceof MqttMessageIdVariableHeader messageIdVariableHeader) { + originalMessageId = String.valueOf(messageIdVariableHeader.messageId()); + } else if (variableHeader instanceof MqttPublishVariableHeader publishVariableHeader) { + originalMessageId = String.valueOf(publishVariableHeader.packetId()); + } else { + originalMessageId = "N/A"; + } + } + void start(EventLoop eventLoop) { if (eventLoop == null) { throw new NullPointerException("eventLoop"); } - if (this.handler == null) { + if (handler == null) { throw new NullPointerException("handler"); } - this.timeout = 10; - this.startTimer(eventLoop); + log.debug("{}MessageID[{}] Starting retransmission handler", ownerId, originalMessageId); + startTimer(eventLoop); } private void startTimer(EventLoop eventLoop) { - if (stopped || pendingOperation.isCanceled()) { + if (stopped || pendingOperation.isCancelled()) { return; } - this.timer = eventLoop.schedule(() -> { - if (stopped || pendingOperation.isCanceled()) { + + // Calculate the base delay using exponential backoff. + // For attemptCount == 0, delay = initial delay; for each subsequent attempt, the base delay doubles. + long baseDelay = config.initialDelayMillis() * (long) Math.pow(2, attemptCount); + // Apply jitter: random factor between (1 - jitterFactor) and (1 + jitterFactor). + double minFactor = 1.0 - config.jitterFactor(); + double maxFactor = 1.0 + config.jitterFactor(); + double randomFactor = config.jitterFactor() == 0 ? 1 : ThreadLocalRandom.current().nextDouble(minFactor, maxFactor); + long delayMillisWithJitter = (long) (baseDelay * randomFactor); + totalWaitingTimeMillis += delayMillisWithJitter; + + timer = eventLoop.schedule(() -> { + if (stopped || pendingOperation.isCancelled()) { return; } - this.timeout += 5; - boolean isDup = this.originalMessage.fixedHeader().isDup(); - if (this.originalMessage.fixedHeader().messageType() == MqttMessageType.PUBLISH && this.originalMessage.fixedHeader().qosLevel() != MqttQoS.AT_MOST_ONCE) { - isDup = true; + + attemptCount++; + if (attemptCount > config.maxAttempts()) { + log.debug( + "{}MessageID[{}] Gave up after {} retransmission attempts; waited a total of {} ms without receiving acknowledgement", + ownerId, originalMessageId, config.maxAttempts(), totalWaitingTimeMillis + ); + stop(); + pendingOperation.onMaxRetransmissionAttemptsReached(); + return; } - MqttFixedHeader fixedHeader = new MqttFixedHeader(this.originalMessage.fixedHeader().messageType(), isDup, this.originalMessage.fixedHeader().qosLevel(), this.originalMessage.fixedHeader().isRetain(), this.originalMessage.fixedHeader().remainingLength()); - handler.accept(fixedHeader, originalMessage); + + log.debug("{}MessageID[{}] Retransmission attempt #{} out of {}", ownerId, originalMessageId, attemptCount, config.maxAttempts()); + + var originalFixedHeader = originalMessage.fixedHeader(); + var newFixedHeader = new MqttFixedHeader( + originalFixedHeader.messageType(), + isDup(originalFixedHeader), + originalFixedHeader.qosLevel(), + originalFixedHeader.isRetain(), + originalFixedHeader.remainingLength() + ); + handler.accept(newFixedHeader, originalMessage); startTimer(eventLoop); - }, timeout, TimeUnit.SECONDS); + }, delayMillisWithJitter, TimeUnit.MILLISECONDS); + } + + private static boolean isDup(MqttFixedHeader originalFixedHeader) { + return originalFixedHeader.isDup() || (originalFixedHeader.messageType() == MqttMessageType.PUBLISH && originalFixedHeader.qosLevel() != MqttQoS.AT_MOST_ONCE); } void stop() { + log.debug("{}MessageID[{}] Stopping retransmission handler", ownerId, originalMessageId); stopped = true; - if (this.timer != null) { - this.timer.cancel(true); + if (timer != null) { + timer.cancel(true); } } - void setHandle(BiConsumer runnable) { - this.handler = runnable; - } - - void setOriginalMessage(T originalMessage) { - this.originalMessage = originalMessage; - } } diff --git a/netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttClientTest.java b/netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttClientTest.java new file mode 100644 index 0000000000..1481b354ee --- /dev/null +++ b/netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttClientTest.java @@ -0,0 +1,210 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt; + +import com.google.common.util.concurrent.Futures; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.PooledByteBufAllocator; +import io.netty.handler.codec.mqtt.MqttConnectReturnCode; +import io.netty.handler.codec.mqtt.MqttMessageType; +import io.netty.handler.codec.mqtt.MqttQoS; +import io.netty.util.ResourceLeakDetector; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.Promise; +import lombok.extern.slf4j.Slf4j; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.hivemq.HiveMQContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; +import org.thingsboard.common.util.AbstractListeningExecutor; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +@Slf4j +@Testcontainers +class MqttClientTest { + + final int randomPort = 0; + + @Container + HiveMQContainer broker = new HiveMQContainer(DockerImageName.parse("hivemq/hivemq-ce").withTag("2025.2")); + + MqttTestProxy proxy; + + MqttClient client; + + AbstractListeningExecutor handlerExecutor; + + @BeforeAll + static void init() { + ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID); + } + + @BeforeEach + void setup() { + handlerExecutor = new AbstractListeningExecutor() { + @Override + protected int getThreadPollSize() { + return 1; + } + }; + handlerExecutor.init(); + } + + @AfterEach + void cleanup() { + if (client != null) { + client.disconnect(); + client = null; + } + if (proxy != null) { + proxy.stop(); + proxy = null; + } + handlerExecutor.destroy(); + handlerExecutor = null; + } + + @Test + void testConnectToBroker() { + // GIVEN + var clientConfig = new MqttClientConfig(); + clientConfig.setOwnerId("Test[ConnectToBroker]"); + clientConfig.setClientId("connect"); + + client = MqttClient.create(clientConfig, null, handlerExecutor); + + // WHEN + Promise connectFuture = client.connect(broker.getHost(), broker.getMqttPort()); + + // THEN + assertThat(connectFuture).isNotNull(); + + Awaitility.await("waiting for client to connect") + .atMost(Duration.ofSeconds(10L)) + .until(connectFuture::isDone); + + assertThat(connectFuture.isSuccess()).isTrue(); + + MqttConnectResult actualConnectResult = connectFuture.getNow(); + assertThat(actualConnectResult).isNotNull(); + assertThat(actualConnectResult.isSuccess()).isTrue(); + assertThat(actualConnectResult.getReturnCode()).isEqualTo(MqttConnectReturnCode.CONNECTION_ACCEPTED); + + assertThat(client.isConnected()).isTrue(); + } + + @Test + void testDisconnectDueToKeepAliveIfNoActivity() { + // GIVEN + proxy = MqttTestProxy.builder() + .localPort(randomPort) + .brokerHost(broker.getHost()) + .brokerPort(broker.getMqttPort()) + .brokerToClientInterceptor(msg -> msg.fixedHeader().messageType() != MqttMessageType.PINGRESP) // drop all ping responses to simulate broker down + .build(); + + int idleTimeoutSeconds = 2; + + var clientConfig = new MqttClientConfig(); + clientConfig.setOwnerId("Test[KeepAliveDisconnect]"); + clientConfig.setClientId("no-activity-disconnect"); + clientConfig.setTimeoutSeconds(idleTimeoutSeconds); + clientConfig.setReconnect(false); // disable auto reconnect + client = MqttClient.create(clientConfig, null, handlerExecutor); + + // WHEN-THEN + connect(broker.getHost(), proxy.getPort()); + + // no activity... + + Awaitility.await("waiting for client to disconnect") + .pollDelay(Duration.ofSeconds(idleTimeoutSeconds * 2)) // 2 seconds to wait for the first idle event and then 2 seconds for scheduled disconnect to fire + .atMost(Duration.ofSeconds(10)) + .untilAsserted(() -> assertThat(client.isConnected()).isFalse()); + } + + @Test + void testRetransmission() { + // GIVEN + proxy = MqttTestProxy.builder() + .localPort(randomPort) + .brokerHost(broker.getHost()) + .brokerPort(broker.getMqttPort()) + .brokerToClientInterceptor(msg -> msg.fixedHeader().messageType() != MqttMessageType.PUBACK) // drop all pubacks to allow retransmission to happen + .build(); + + // create client + var clientConfig = new MqttClientConfig(); + clientConfig.setOwnerId("Test[Retransmission]"); + clientConfig.setClientId("retransmission"); + clientConfig.setRetransmissionConfig(new MqttClientConfig.RetransmissionConfig(1, 1000L, 0d)); + client = MqttClient.create(clientConfig, null, handlerExecutor); + + // connect to a broker + connect(broker.getHost(), proxy.getPort()); + + // subscribe to a topic + String topic = "test-topic"; + List receivedMessages = Collections.synchronizedList(new ArrayList<>(2)); + Future subscribeFuture = client.on(topic, (__, payload) -> { + receivedMessages.add(payload); + return Futures.immediateVoidFuture(); + }); + Awaitility.await("waiting for client to subscribe to a topic") + .atMost(Duration.ofSeconds(10L)) + .until(subscribeFuture::isDone); + + // WHEN + // publish a message + ByteBuf message = PooledByteBufAllocator.DEFAULT.buffer().writeBytes("test message".getBytes(StandardCharsets.UTF_8)); + client.publish(topic, message, MqttQoS.AT_LEAST_ONCE); + + // THEN + // wait enough time so that retransmission happens and stops + // if retransmission works incorrectly waiting 10 seconds allows for additional retransmissions to happen + try { + Awaitility.await("wait up to 10s, stop early if too many messages") + .atMost(Duration.ofSeconds(10L)) + .pollInterval(Duration.ofMillis(100)) + .until(() -> receivedMessages.size() > 2); + } catch (ConditionTimeoutException __) { + // didn't exceed 2 messages + } + + assertThat(receivedMessages).size().describedAs("incorrect number of messages received, expected 2 (original plus one retransmitted)").isEqualTo(2); + } + + private void connect(String host, int port) { + Promise connectFuture = client.connect(host, port); + Awaitility.await("waiting for client to connect") + .atMost(Duration.ofSeconds(10L)) + .until(connectFuture::isSuccess); + } + +} diff --git a/netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttPingHandlerTest.java b/netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttPingHandlerTest.java deleted file mode 100644 index 83e3b1c8d5..0000000000 --- a/netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttPingHandlerTest.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.mqtt; - -import io.netty.channel.Channel; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.DefaultEventLoop; -import io.netty.handler.timeout.IdleStateEvent; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.util.concurrent.TimeUnit; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.after; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -class MqttPingHandlerTest { - - static final int KEEP_ALIVE_SECONDS = 0; - static final int PROCESS_SEND_DISCONNECT_MSG_TIME_MS = 500; - - MqttPingHandler mqttPingHandler; - - @BeforeEach - void setUp() { - mqttPingHandler = new MqttPingHandler(KEEP_ALIVE_SECONDS); - } - - @Test - void givenChannelReaderIdleState_whenNoPingResponse_thenDisconnectClient() throws Exception { - ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); - Channel channel = mock(Channel.class); - when(ctx.channel()).thenReturn(channel); - when(channel.eventLoop()).thenReturn(new DefaultEventLoop()); - ChannelFuture channelFuture = mock(ChannelFuture.class); - when(channel.writeAndFlush(any())).thenReturn(channelFuture); - - mqttPingHandler.userEventTriggered(ctx, IdleStateEvent.FIRST_READER_IDLE_STATE_EVENT); - verify( - channelFuture, - after(TimeUnit.SECONDS.toMillis(KEEP_ALIVE_SECONDS) + PROCESS_SEND_DISCONNECT_MSG_TIME_MS) - ).addListener(eq(ChannelFutureListener.CLOSE)); - } -} \ No newline at end of file diff --git a/netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttTestProxy.java b/netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttTestProxy.java new file mode 100644 index 0000000000..4a10fc3bfb --- /dev/null +++ b/netty-mqtt/src/test/java/org/thingsboard/mqtt/MqttTestProxy.java @@ -0,0 +1,202 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.mqtt; + +import io.netty.bootstrap.Bootstrap; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.handler.codec.mqtt.MqttDecoder; +import io.netty.handler.codec.mqtt.MqttEncoder; +import io.netty.handler.codec.mqtt.MqttMessage; +import io.netty.util.ReferenceCountUtil; +import lombok.extern.slf4j.Slf4j; + +import java.net.InetSocketAddress; +import java.util.function.Predicate; + +@Slf4j +public class MqttTestProxy { + + private final EventLoopGroup bossGroup; + private final EventLoopGroup workerGroup; + + private Channel clientToProxyChannel; + private Channel proxyToBrokerChannel; + + private final int assignedPort; + + private boolean stopped; + + private final Predicate brokerToClientInterceptor; + + private MqttTestProxy(Builder builder) { + log.info("Starting MQTT proxy..."); + + brokerToClientInterceptor = builder.brokerToClientInterceptor != null ? builder.brokerToClientInterceptor : msg -> true; + bossGroup = new NioEventLoopGroup(1); + workerGroup = new NioEventLoopGroup(1); + + ServerBootstrap proxyBootstrap = new ServerBootstrap(); + proxyBootstrap.group(bossGroup, workerGroup) + .channel(NioServerSocketChannel.class) + .childHandler(new ChannelInitializer() { + @Override + protected void initChannel(SocketChannel channel) { + clientToProxyChannel = channel; + clientToProxyChannel.config().setAutoRead(false); // do not accept data before we connected to a broker + + connectToBroker(builder.brokerHost, builder.brokerPort).addListener(future -> { + if (future.isSuccess()) { + clientToProxyChannel.pipeline().addLast("mqttDecoder", new MqttDecoder()); + clientToProxyChannel.pipeline().addLast("mqttToBroker", new MqttRelayHandler(proxyToBrokerChannel, null)); + clientToProxyChannel.pipeline().addLast("mqttEncoder", MqttEncoder.INSTANCE); + + clientToProxyChannel.config().setAutoRead(true); // start accepting data for a client + } else { + log.error("Failed to connect to broker", future.cause()); + clientToProxyChannel.close(); + } + }); + } + }); + + try { + Channel proxyChannel = proxyBootstrap.bind(builder.localPort).sync().channel(); + assignedPort = ((InetSocketAddress) proxyChannel.localAddress()).getPort(); + } catch (Exception e) { + log.error("Failed to start MQTT proxy", e); + throw new RuntimeException("Failed to start MQTT proxy", e); + } + + log.info("MQTT proxy started on port {}", assignedPort); + } + + private ChannelFuture connectToBroker(String brokerHost, int brokerPort) { + Bootstrap proxyToBrokerBootstrap = new Bootstrap(); + proxyToBrokerBootstrap.group(workerGroup) + .channel(NioSocketChannel.class) + .handler(new ChannelInitializer() { + @Override + protected void initChannel(SocketChannel channel) { + proxyToBrokerChannel = channel; + proxyToBrokerChannel.pipeline().addLast(new MqttDecoder()); + proxyToBrokerChannel.pipeline().addLast("mqttToClient", new MqttRelayHandler(clientToProxyChannel, brokerToClientInterceptor)); + proxyToBrokerChannel.pipeline().addLast(MqttEncoder.INSTANCE); + } + }); + return proxyToBrokerBootstrap.connect(brokerHost, brokerPort); + } + + private static class MqttRelayHandler extends SimpleChannelInboundHandler { + + private final Channel targetChannel; + private final Predicate interceptor; + + private MqttRelayHandler(Channel targetChannel, Predicate interceptor) { + this.targetChannel = targetChannel; + this.interceptor = interceptor; + } + + @Override + protected void channelRead0(ChannelHandlerContext ctx, MqttMessage msg) { + log.debug("Received message: {}", msg.fixedHeader().messageType()); + if (interceptor == null || interceptor.test(msg)) { + if (targetChannel.isActive()) { + targetChannel.writeAndFlush(ReferenceCountUtil.retain(msg)); + } + } else { + log.info("Dropping message: {}", msg.fixedHeader().messageType()); + } + } + + } + + public void stop() { + if (stopped) { + log.info("MQTT proxy was already stopped"); + return; + } + + stopped = true; + + log.info("Stopping MQTT proxy..."); + + if (clientToProxyChannel != null) { + clientToProxyChannel.close(); + } + if (proxyToBrokerChannel != null) { + proxyToBrokerChannel.close(); + } + if (bossGroup != null) { + bossGroup.shutdownGracefully(); + } + if (workerGroup != null) { + workerGroup.shutdownGracefully(); + } + + log.info("MQTT proxy stopped"); + } + + public int getPort() { + return assignedPort; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + + private int localPort; + private String brokerHost; + private int brokerPort; + private Predicate brokerToClientInterceptor; + + public Builder localPort(int localPort) { + this.localPort = localPort; + return this; + } + + public Builder brokerHost(String brokerHost) { + this.brokerHost = brokerHost; + return this; + } + + public Builder brokerPort(int brokerPort) { + this.brokerPort = brokerPort; + return this; + } + + public Builder brokerToClientInterceptor(Predicate interceptor) { + this.brokerToClientInterceptor = interceptor; + return this; + } + + public MqttTestProxy build() { + return new MqttTestProxy(this); + } + + } +} diff --git a/netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/MqttIntegrationTest.java b/netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/MqttIntegrationTest.java deleted file mode 100644 index db177c84b6..0000000000 --- a/netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/MqttIntegrationTest.java +++ /dev/null @@ -1,151 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.mqtt.integration; - -import io.netty.buffer.Unpooled; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.handler.codec.mqtt.MqttMessageType; -import io.netty.handler.codec.mqtt.MqttQoS; -import io.netty.util.concurrent.Future; -import io.netty.util.concurrent.Promise; -import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.parallel.ResourceLock; -import org.thingsboard.common.util.AbstractListeningExecutor; -import org.thingsboard.mqtt.MqttClient; -import org.thingsboard.mqtt.MqttClientConfig; -import org.thingsboard.mqtt.MqttConnectResult; -import org.thingsboard.mqtt.integration.server.MqttServer; - -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -@ResourceLock("port8885") // test MQTT server port -@Slf4j -public class MqttIntegrationTest { - - static final String MQTT_HOST = "localhost"; - static final int KEEPALIVE_TIMEOUT_SECONDS = 2; - static final long RECONNECT_DELAY_SECONDS = 10L; - - EventLoopGroup eventLoopGroup; - MqttServer mqttServer; - - MqttClient mqttClient; - - AbstractListeningExecutor handlerExecutor; - - @BeforeEach - public void init() throws Exception { - this.handlerExecutor = new AbstractListeningExecutor() { - @Override - protected int getThreadPollSize() { - return 4; - } - }; - handlerExecutor.init(); - - this.eventLoopGroup = new NioEventLoopGroup(); - - this.mqttServer = new MqttServer(); - this.mqttServer.init(); - } - - @AfterEach - public void destroy() throws InterruptedException { - if (this.mqttClient != null) { - this.mqttClient.disconnect(); - } - if (this.mqttServer != null) { - this.mqttServer.shutdown(); - } - if (this.eventLoopGroup != null) { - this.eventLoopGroup.shutdownGracefully(0, 0, TimeUnit.MILLISECONDS); - } - if (this.handlerExecutor != null) { - this.handlerExecutor.destroy(); - } - } - - @Test - public void givenActiveMqttClient_whenNoActivityForKeepAliveTimeout_thenDisconnectClient() throws Throwable { - //given - this.mqttClient = initClient(); - - log.warn("Sending publish messages..."); - CountDownLatch latch = new CountDownLatch(3); - for (int i = 0; i < 3; i++) { - Thread.sleep(30); - Future pubFuture = publishMsg(); - pubFuture.addListener(future -> latch.countDown()); - } - - log.warn("Waiting for messages acknowledgments..."); - boolean awaitResult = latch.await(10, TimeUnit.SECONDS); - Assertions.assertTrue(awaitResult); - log.warn("Messages are delivered successfully..."); - - //when - log.warn("Starting idle period..."); - Thread.sleep(5000); - - //then - List allReceivedEvents = this.mqttServer.getEventsFromClient(); - long disconnectCount = allReceivedEvents.stream().filter(type -> type == MqttMessageType.DISCONNECT).count(); - - Assertions.assertEquals(1, disconnectCount); - } - - private Future publishMsg() { - return this.mqttClient.publish( - "test/topic", - Unpooled.wrappedBuffer("payload".getBytes(StandardCharsets.UTF_8)), - MqttQoS.AT_MOST_ONCE); - } - - private MqttClient initClient() throws Exception { - MqttClientConfig config = new MqttClientConfig(); - config.setOwnerId("MqttIntegrationTest"); - config.setTimeoutSeconds(KEEPALIVE_TIMEOUT_SECONDS); - config.setReconnectDelay(RECONNECT_DELAY_SECONDS); - MqttClient client = MqttClient.create(config, null, handlerExecutor); - client.setEventLoop(this.eventLoopGroup); - Promise connectFuture = client.connect(MQTT_HOST, this.mqttServer.getMqttPort()); - - String hostPort = MQTT_HOST + ":" + this.mqttServer.getMqttPort(); - MqttConnectResult result; - try { - result = connectFuture.get(10, TimeUnit.SECONDS); - } catch (TimeoutException ex) { - connectFuture.cancel(true); - client.disconnect(); - throw new RuntimeException(String.format("Failed to connect to MQTT server at %s.", hostPort)); - } - if (!result.isSuccess()) { - connectFuture.cancel(true); - client.disconnect(); - throw new RuntimeException(String.format("Failed to connect to MQTT server at %s. Result code is: %s", hostPort, result.getReturnCode())); - } - return client; - } -} \ No newline at end of file diff --git a/netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/server/MqttServer.java b/netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/server/MqttServer.java deleted file mode 100644 index ca4fb677dc..0000000000 --- a/netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/server/MqttServer.java +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.mqtt.integration.server; - -import io.netty.bootstrap.ServerBootstrap; -import io.netty.channel.Channel; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.ChannelPipeline; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.SocketChannel; -import io.netty.channel.socket.nio.NioServerSocketChannel; -import io.netty.handler.codec.mqtt.MqttDecoder; -import io.netty.handler.codec.mqtt.MqttEncoder; -import io.netty.handler.codec.mqtt.MqttMessageType; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; - -@Slf4j -public class MqttServer { - - @Getter - private final List eventsFromClient = new CopyOnWriteArrayList<>(); - @Getter - private final int mqttPort = 8885; - - private Channel serverChannel; - private EventLoopGroup bossGroup; - private EventLoopGroup workerGroup; - - public void init() throws Exception { - log.info("Starting MQTT server on port {}...", mqttPort); - bossGroup = new NioEventLoopGroup(); - workerGroup = new NioEventLoopGroup(); - ServerBootstrap b = new ServerBootstrap(); - b.group(bossGroup, workerGroup) - .channel(NioServerSocketChannel.class) - .childHandler(new ChannelInitializer() { - @Override - protected void initChannel(SocketChannel ch) throws Exception { - ChannelPipeline pipeline = ch.pipeline(); - pipeline.addLast("decoder", new MqttDecoder(65536)); - pipeline.addLast("encoder", MqttEncoder.INSTANCE); - - MqttTransportHandler handler = new MqttTransportHandler(eventsFromClient); - - pipeline.addLast(handler); - ch.closeFuture().addListener(handler); - } - }) - .childOption(ChannelOption.SO_KEEPALIVE, true); - - serverChannel = b.bind(mqttPort).sync().channel(); - log.info("Mqtt transport started!"); - } - - public void shutdown() throws InterruptedException { - log.info("Stopping MQTT transport!"); - try { - serverChannel.close().sync(); - } finally { - workerGroup.shutdownGracefully(); - bossGroup.shutdownGracefully(); - } - log.info("MQTT transport stopped!"); - } -} diff --git a/netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/server/MqttTransportHandler.java b/netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/server/MqttTransportHandler.java deleted file mode 100644 index 5c433d7069..0000000000 --- a/netty-mqtt/src/test/java/org/thingsboard/mqtt/integration/server/MqttTransportHandler.java +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.mqtt.integration.server; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.handler.codec.mqtt.MqttConnAckMessage; -import io.netty.handler.codec.mqtt.MqttConnAckVariableHeader; -import io.netty.handler.codec.mqtt.MqttConnectMessage; -import io.netty.handler.codec.mqtt.MqttConnectReturnCode; -import io.netty.handler.codec.mqtt.MqttFixedHeader; -import io.netty.handler.codec.mqtt.MqttMessage; -import io.netty.handler.codec.mqtt.MqttMessageIdVariableHeader; -import io.netty.handler.codec.mqtt.MqttMessageType; -import io.netty.handler.codec.mqtt.MqttPubAckMessage; -import io.netty.handler.codec.mqtt.MqttPublishMessage; -import io.netty.util.ReferenceCountUtil; -import io.netty.util.concurrent.Future; -import io.netty.util.concurrent.GenericFutureListener; -import lombok.extern.slf4j.Slf4j; - -import java.util.List; -import java.util.UUID; - -import static io.netty.handler.codec.mqtt.MqttMessageType.CONNACK; -import static io.netty.handler.codec.mqtt.MqttMessageType.CONNECT; -import static io.netty.handler.codec.mqtt.MqttMessageType.DISCONNECT; -import static io.netty.handler.codec.mqtt.MqttMessageType.PINGREQ; -import static io.netty.handler.codec.mqtt.MqttMessageType.PUBACK; -import static io.netty.handler.codec.mqtt.MqttMessageType.PUBLISH; -import static io.netty.handler.codec.mqtt.MqttQoS.AT_MOST_ONCE; - -@Slf4j -public class MqttTransportHandler extends ChannelInboundHandlerAdapter implements GenericFutureListener> { - - private final List eventsFromClient; - private final UUID sessionId; - - MqttTransportHandler(List eventsFromClient) { - this.sessionId = UUID.randomUUID(); - this.eventsFromClient = eventsFromClient; - } - - @Override - public void channelRead(ChannelHandlerContext ctx, Object msg) { - log.trace("[{}] Processing msg: {}", sessionId, msg); - try { - if (msg instanceof MqttMessage) { - MqttMessage message = (MqttMessage) msg; - if (message.decoderResult().isSuccess()) { - processMqttMsg(ctx, message); - } else { - log.error("[{}] Message decoding failed: {}", sessionId, message.decoderResult().cause().getMessage()); - ctx.close(); - } - } else { - log.debug("[{}] Received non mqtt message: {}", sessionId, msg.getClass().getSimpleName()); - ctx.close(); - } - } finally { - ReferenceCountUtil.safeRelease(msg); - } - } - - void processMqttMsg(ChannelHandlerContext ctx, MqttMessage msg) { - if (msg.fixedHeader() == null) { - ctx.close(); - return; - } - switch (msg.fixedHeader().messageType()) { - case CONNECT: - eventsFromClient.add(CONNECT); - processConnect(ctx, (MqttConnectMessage) msg); - break; - case DISCONNECT: - eventsFromClient.add(DISCONNECT); - ctx.close(); - break; - case PUBLISH: - // QoS 0 and 1 supported only here - eventsFromClient.add(PUBLISH); - MqttPublishMessage mqttPubMsg = (MqttPublishMessage) msg; - ack(ctx, mqttPubMsg.variableHeader().packetId()); - break; - case PINGREQ: - // We will not handle PINGREQ and will not send any PINGRESP to simulate the MQTT server is down - eventsFromClient.add(PINGREQ); - break; - default: - break; - } - } - - void processConnect(ChannelHandlerContext ctx, MqttConnectMessage msg) { - String userName = msg.payload().userName(); - String clientId = msg.payload().clientIdentifier(); - - log.warn("[{}][{}] Processing connect msg for client: {}!", sessionId, userName, clientId); - ctx.writeAndFlush(createMqttConnAckMsg(msg)); - } - - private MqttConnAckMessage createMqttConnAckMsg(MqttConnectMessage msg) { - MqttFixedHeader mqttFixedHeader = - new MqttFixedHeader(CONNACK, false, AT_MOST_ONCE, false, 0); - MqttConnAckVariableHeader mqttConnAckVariableHeader = - new MqttConnAckVariableHeader(MqttConnectReturnCode.CONNECTION_ACCEPTED, !msg.variableHeader().isCleanSession()); - return new MqttConnAckMessage(mqttFixedHeader, mqttConnAckVariableHeader); - } - - private void ack(ChannelHandlerContext ctx, int msgId) { - if (msgId > 0) { - ctx.writeAndFlush(createMqttPubAckMsg(msgId)); - } - } - - public static MqttPubAckMessage createMqttPubAckMsg(int requestId) { - MqttFixedHeader mqttFixedHeader = - new MqttFixedHeader(PUBACK, false, AT_MOST_ONCE, false, 0); - MqttMessageIdVariableHeader mqttMsgIdVariableHeader = - MqttMessageIdVariableHeader.from(requestId); - return new MqttPubAckMessage(mqttFixedHeader, mqttMsgIdVariableHeader); - } - - @Override - public void operationComplete(Future future) { - log.trace("[{}] Channel closed!", sessionId); - } -} diff --git a/netty-mqtt/src/test/resources/junit-platform.properties b/netty-mqtt/src/test/resources/junit-platform.properties deleted file mode 100644 index f2ed301920..0000000000 --- a/netty-mqtt/src/test/resources/junit-platform.properties +++ /dev/null @@ -1,3 +0,0 @@ -junit.jupiter.execution.parallel.enabled = true -junit.jupiter.execution.parallel.mode.default = concurrent -junit.jupiter.execution.parallel.mode.classes.default = concurrent diff --git a/pom.xml b/pom.xml index 72195b1e4f..70e0777462 100755 --- a/pom.xml +++ b/pom.xml @@ -1957,6 +1957,12 @@ ${testcontainers.version} test + + org.testcontainers + hivemq + ${testcontainers.version} + test + org.springframework.data spring-data-redis diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/MqttClientSettings.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/MqttClientSettings.java new file mode 100644 index 0000000000..4ac05b57d0 --- /dev/null +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/MqttClientSettings.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.api; + +public interface MqttClientSettings { + + int getRetransmissionMaxAttempts(); + + long getRetransmissionInitialDelayMillis(); + + double getRetransmissionJitterFactor(); + +} diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index b66c9e13d5..7989b8f9ce 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -416,4 +416,9 @@ public interface TbContext { EventService getEventService(); AuditLogService getAuditLogService(); + + // Configuration parameters for the MQTT client that is used in the MQTT node and Azure IoT hub node + + MqttClientSettings getMqttClientSettings(); + } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java index 4d99951e1a..28a9e1ff4b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/mqtt/TbMqttNode.java @@ -26,6 +26,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.mqtt.MqttClient; import org.thingsboard.mqtt.MqttClientConfig; import org.thingsboard.mqtt.MqttConnectResult; +import org.thingsboard.rule.engine.api.MqttClientSettings; import org.thingsboard.rule.engine.api.RuleNode; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeConfiguration; @@ -126,6 +127,13 @@ public class TbMqttNode extends TbAbstractExternalNode { } config.setCleanSession(this.mqttNodeConfiguration.isCleanSession()); + MqttClientSettings mqttClientSettings = ctx.getMqttClientSettings(); + config.setRetransmissionConfig(new MqttClientConfig.RetransmissionConfig( + mqttClientSettings.getRetransmissionMaxAttempts(), + mqttClientSettings.getRetransmissionInitialDelayMillis(), + mqttClientSettings.getRetransmissionJitterFactor() + )); + prepareMqttClientConfig(config); MqttClient client = getMqttClient(ctx, config); client.setEventLoop(ctx.getSharedEventLoop()); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mqtt/TbMqttNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mqtt/TbMqttNodeTest.java index f6ccfbca6f..bf650af8bb 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mqtt/TbMqttNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mqtt/TbMqttNodeTest.java @@ -40,6 +40,7 @@ import org.thingsboard.mqtt.MqttClient; import org.thingsboard.mqtt.MqttClientConfig; import org.thingsboard.mqtt.MqttConnectResult; import org.thingsboard.rule.engine.AbstractRuleNodeUpgradeTest; +import org.thingsboard.rule.engine.api.MqttClientSettings; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; @@ -80,6 +81,7 @@ import static org.mockito.BDDMockito.spy; import static org.mockito.BDDMockito.then; import static org.mockito.BDDMockito.willAnswer; import static org.mockito.BDDMockito.willReturn; +import static org.mockito.Mockito.lenient; @ExtendWith(MockitoExtension.class) public class TbMqttNodeTest extends AbstractRuleNodeUpgradeTest { @@ -106,6 +108,22 @@ public class TbMqttNodeTest extends AbstractRuleNodeUpgradeTest { protected void setUp() { mqttNode = spy(new TbMqttNode()); mqttNodeConfig = new TbMqttNodeConfiguration().defaultConfiguration(); + lenient().when(ctxMock.getMqttClientSettings()).thenReturn(new MqttClientSettings() { + @Override + public int getRetransmissionMaxAttempts() { + return 3; + } + + @Override + public long getRetransmissionInitialDelayMillis() { + return 5000L; + } + + @Override + public double getRetransmissionJitterFactor() { + return 0.15; + } + }); } @Test From 777228afe4d235b4b835403cbd2c80a0e4e2dbc2 Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Thu, 1 May 2025 16:39:46 +0300 Subject: [PATCH 17/79] UI: Rename id property --- .../system/scada_symbols/bottom-right-elbow-connector-hp.svg | 4 ++-- .../json/system/scada_symbols/bottom-tee-connector-hp.svg | 4 ++-- .../data/json/system/scada_symbols/cross-connector-hp.svg | 2 +- .../json/system/scada_symbols/horizontal-connector-hp.svg | 4 ++-- .../system/scada_symbols/left-bottom-elbow-connector-hp.svg | 4 ++-- .../data/json/system/scada_symbols/left-tee-connector-hp.svg | 4 ++-- .../json/system/scada_symbols/left-top-elbow-connector-hp.svg | 4 ++-- .../system/scada_symbols/long-horizontal-connector-hp.svg | 4 ++-- .../json/system/scada_symbols/long-vertical-connector-hp.svg | 4 ++-- .../data/json/system/scada_symbols/right-tee-connector-hp.svg | 4 ++-- .../system/scada_symbols/top-right-elbow-connector-hp.svg | 4 ++-- .../data/json/system/scada_symbols/top-tee-connector-hp.svg | 4 ++-- .../data/json/system/scada_symbols/vertical-connector-hp.svg | 4 ++-- 13 files changed, 25 insertions(+), 25 deletions(-) diff --git a/application/src/main/data/json/system/scada_symbols/bottom-right-elbow-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/bottom-right-elbow-connector-hp.svg index f7ffd5f167..a0a56e37e5 100644 --- a/application/src/main/data/json/system/scada_symbols/bottom-right-elbow-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/bottom-right-elbow-connector-hp.svg @@ -7,7 +7,7 @@ "tags": [ { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.mainLineSize});", "actions": null } ], @@ -131,7 +131,7 @@ ], "properties": [ { - "id": "lineSize", + "id": "mainLineSize", "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, diff --git a/application/src/main/data/json/system/scada_symbols/bottom-tee-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/bottom-tee-connector-hp.svg index bdbbdca81f..90cfe6ed2b 100644 --- a/application/src/main/data/json/system/scada_symbols/bottom-tee-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/bottom-tee-connector-hp.svg @@ -7,7 +7,7 @@ "tags": [ { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.mainLineSize});", "actions": null }, { @@ -368,7 +368,7 @@ ], "properties": [ { - "id": "lineSize", + "id": "mainLineSize", "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, diff --git a/application/src/main/data/json/system/scada_symbols/cross-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/cross-connector-hp.svg index db6b091e0e..5ac0af8248 100644 --- a/application/src/main/data/json/system/scada_symbols/cross-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/cross-connector-hp.svg @@ -7,7 +7,7 @@ "tags": [ { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.mainLineSize});", "actions": null }, { diff --git a/application/src/main/data/json/system/scada_symbols/horizontal-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/horizontal-connector-hp.svg index 93ef81f57b..f2fc9fb2e0 100644 --- a/application/src/main/data/json/system/scada_symbols/horizontal-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/horizontal-connector-hp.svg @@ -12,7 +12,7 @@ }, { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.mainLineSize});", "actions": null } ], @@ -167,7 +167,7 @@ ], "properties": [ { - "id": "lineSize", + "id": "mainLineSize", "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, diff --git a/application/src/main/data/json/system/scada_symbols/left-bottom-elbow-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/left-bottom-elbow-connector-hp.svg index 1c24cca2e0..8fd4ee09df 100644 --- a/application/src/main/data/json/system/scada_symbols/left-bottom-elbow-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/left-bottom-elbow-connector-hp.svg @@ -7,7 +7,7 @@ "tags": [ { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.mainLineSize});", "actions": null } ], @@ -131,7 +131,7 @@ ], "properties": [ { - "id": "lineSize", + "id": "mainLineSize", "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, diff --git a/application/src/main/data/json/system/scada_symbols/left-tee-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/left-tee-connector-hp.svg index 461e91fe94..c3b4e67d56 100644 --- a/application/src/main/data/json/system/scada_symbols/left-tee-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/left-tee-connector-hp.svg @@ -7,7 +7,7 @@ "tags": [ { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.mainLineSize});", "actions": null }, { @@ -368,7 +368,7 @@ ], "properties": [ { - "id": "lineSize", + "id": "mainLineSize", "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, diff --git a/application/src/main/data/json/system/scada_symbols/left-top-elbow-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/left-top-elbow-connector-hp.svg index 19e217a428..961c760f63 100644 --- a/application/src/main/data/json/system/scada_symbols/left-top-elbow-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/left-top-elbow-connector-hp.svg @@ -7,7 +7,7 @@ "tags": [ { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.mainLineSize});", "actions": null } ], @@ -131,7 +131,7 @@ ], "properties": [ { - "id": "lineSize", + "id": "mainLineSize", "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, diff --git a/application/src/main/data/json/system/scada_symbols/long-horizontal-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/long-horizontal-connector-hp.svg index 0fe805c9a7..f5f68c4934 100644 --- a/application/src/main/data/json/system/scada_symbols/long-horizontal-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/long-horizontal-connector-hp.svg @@ -12,7 +12,7 @@ }, { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.mainLineSize});", "actions": null } ], @@ -167,7 +167,7 @@ ], "properties": [ { - "id": "lineSize", + "id": "mainLineSize", "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, diff --git a/application/src/main/data/json/system/scada_symbols/long-vertical-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/long-vertical-connector-hp.svg index 411859b5ed..f344823082 100644 --- a/application/src/main/data/json/system/scada_symbols/long-vertical-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/long-vertical-connector-hp.svg @@ -12,7 +12,7 @@ }, { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.mainLineSize});", "actions": null } ], @@ -167,7 +167,7 @@ ], "properties": [ { - "id": "lineSize", + "id": "mainLineSize", "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, diff --git a/application/src/main/data/json/system/scada_symbols/right-tee-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/right-tee-connector-hp.svg index dafcf53647..346d134f30 100644 --- a/application/src/main/data/json/system/scada_symbols/right-tee-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/right-tee-connector-hp.svg @@ -7,7 +7,7 @@ "tags": [ { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.mainLineSize});", "actions": null }, { @@ -368,7 +368,7 @@ ], "properties": [ { - "id": "lineSize", + "id": "mainLineSize", "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, diff --git a/application/src/main/data/json/system/scada_symbols/top-right-elbow-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/top-right-elbow-connector-hp.svg index f9f2fe20d2..ca255abf14 100644 --- a/application/src/main/data/json/system/scada_symbols/top-right-elbow-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/top-right-elbow-connector-hp.svg @@ -7,7 +7,7 @@ "tags": [ { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.mainLineSize});", "actions": null } ], @@ -131,7 +131,7 @@ ], "properties": [ { - "id": "lineSize", + "id": "mainLineSize", "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, diff --git a/application/src/main/data/json/system/scada_symbols/top-tee-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/top-tee-connector-hp.svg index f61bf85ccd..e63d465d8e 100644 --- a/application/src/main/data/json/system/scada_symbols/top-tee-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/top-tee-connector-hp.svg @@ -7,7 +7,7 @@ "tags": [ { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.mainLineSize});", "actions": null }, { @@ -368,7 +368,7 @@ ], "properties": [ { - "id": "lineSize", + "id": "mainLineSize", "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, diff --git a/application/src/main/data/json/system/scada_symbols/vertical-connector-hp.svg b/application/src/main/data/json/system/scada_symbols/vertical-connector-hp.svg index 9667928e4e..87c3ef23ee 100644 --- a/application/src/main/data/json/system/scada_symbols/vertical-connector-hp.svg +++ b/application/src/main/data/json/system/scada_symbols/vertical-connector-hp.svg @@ -12,7 +12,7 @@ }, { "tag": "line", - "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.lineSize});", + "stateRenderFunction": "element.stroke(ctx.properties.lineColor);\nelement.attr({'stroke-width': ctx.properties.mainLineSize});", "actions": null } ], @@ -167,7 +167,7 @@ ], "properties": [ { - "id": "lineSize", + "id": "mainLineSize", "name": "{i18n:scada.symbol.line}", "type": "number", "default": 6, From 357a1512e863150366fa0f234f9814daf8a57db1 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 2 May 2025 11:26:06 +0300 Subject: [PATCH 18/79] UI: Improved dashboard autocomplete - ignoreError and add error handling --- .../components/dashboard-autocomplete.component.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/shared/components/dashboard-autocomplete.component.ts b/ui-ngx/src/app/shared/components/dashboard-autocomplete.component.ts index 08893efa91..4a74faaf66 100644 --- a/ui-ngx/src/app/shared/components/dashboard-autocomplete.component.ts +++ b/ui-ngx/src/app/shared/components/dashboard-autocomplete.component.ts @@ -190,15 +190,22 @@ export class DashboardAutocompleteComponent implements ControlValueAccessor, OnI this.searchText = ''; if (value != null) { if (typeof value === 'string') { - this.dashboardService.getDashboardInfo(value).subscribe( - (dashboard) => { + this.dashboardService.getDashboardInfo(value, {ignoreLoading: true, ignoreErrors: true}).subscribe({ + next: (dashboard) => { this.modelValue = this.useIdValue ? dashboard.id.id : dashboard; if (this.useDashboardLink) { this.dashboardURL = getEntityDetailsPageURL(this.modelValue as string, EntityType.DASHBOARD); } this.selectDashboardFormGroup.get('dashboard').patchValue(dashboard, {emitEvent: false}); + }, + error: () => { + this.modelValue = null; + this.selectDashboardFormGroup.get('dashboard').patchValue('', {emitEvent: false}); + if (this.required) { + this.propagateChange(this.modelValue); + } } - ); + }); } else { this.modelValue = this.useIdValue ? value.id.id : value; this.selectDashboardFormGroup.get('dashboard').patchValue(value, {emitEvent: false}); From b7081eed0314ca8cee655744096c4f27b99151b7 Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Fri, 2 May 2025 11:27:28 +0300 Subject: [PATCH 19/79] UI: Add completion for new scada api --- .../widget/lib/scada/scada-symbol.models.ts | 2 +- .../scada-symbol-editor.models.ts | 64 +++++++++++++++++++ ui-ngx/src/app/shared/models/constants.ts | 1 + 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/scada/scada-symbol.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/scada/scada-symbol.models.ts index 9e0d07fed4..7bd835de88 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/scada/scada-symbol.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/scada/scada-symbol.models.ts @@ -84,7 +84,7 @@ export interface ScadaSymbolApi { resetCssAnimation: (element: Element) => void; finishCssAnimation: (element: Element) => void; connectorAnimation:(element: Element) => ConnectorScadaSymbolAnimation | undefined; - connectorAnimate:(element: Element) => ConnectorScadaSymbolAnimation; + connectorAnimate:(element: Element, path: string, reversedPath: string) => ConnectorScadaSymbolAnimation; resetConnectorAnimation: (element: Element) => void; finishConnectorAnimation: (element: Element) => void; disable: (element: Element | Element[]) => void; diff --git a/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol-editor.models.ts b/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol-editor.models.ts index 13b55d4f67..56d2801670 100644 --- a/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol-editor.models.ts +++ b/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol-editor.models.ts @@ -1129,6 +1129,8 @@ export const scadaSymbolContextCompletion = (metadata: ScadaSymbolMetadata, tags const scadaSymbolAnimationLink = HelpLinks.linksMap.scadaSymbolDevAnimation; const scadaSymbolAnimation = `ScadaSymbolAnimation`; + const connectorScadaSymbolAnimationLink = HelpLinks.linksMap.scadaSymbolDevConnectorAnimation; + const connectorScadaSymbolAnimation = `ConnectorScadaSymbolAnimation`; const properties: TbEditorCompletion = { meta: 'object', @@ -1229,6 +1231,68 @@ export const scadaSymbolContextCompletion = (metadata: ScadaSymbolMetadata, tags }, ] }, + connectorAnimate: { + meta: 'function', + description: 'Finishes any previous connector animation and starts a new connector animation for the SVG element along the specified path.', + args: [ + { + name: 'element', + description: 'SVG element', + type: 'Element' + }, + { + name: 'path', + description: 'Path defining the animation trajectory', + type: 'string' + }, + { + name: 'reversedPath', + description: 'Path for the reversed animation trajectory', + type: 'string' + } + ], + return: { + description: `Instance of ${connectorScadaSymbolAnimation} class with API to setup and control the connector animation.`, + type: connectorScadaSymbolAnimation + } + }, + connectorAnimation: { + meta: 'function', + description: 'Gets the current connector animation applied to the SVG element.', + args: [ + { + name: 'element', + description: 'SVG element', + type: 'Element' + } + ], + return: { + description: `Instance of ${connectorScadaSymbolAnimation} class with API to setup and control the connector animation, or undefined if no animation is applied.`, + type: 'ConnectorScadaSymbolAnimation | undefined' + } + }, + resetConnectorAnimation: { + meta: 'function', + description: 'Stops the connector animation if any and restores the SVG element to its initial state, removes the connector animation instance.', + args: [ + { + name: 'element', + description: 'SVG element', + type: 'Element' + } + ] + }, + finishConnectorAnimation: { + meta: 'function', + description: 'Finishes the connector animation if any, updates the SVG element state according to the end animation values, removes the connector animation instance.', + args: [ + { + name: 'element', + description: 'SVG element', + type: 'Element' + } + ] + }, generateElementId: { meta: 'function', description: 'Generates new unique element id.', diff --git a/ui-ngx/src/app/shared/models/constants.ts b/ui-ngx/src/app/shared/models/constants.ts index e1306e577c..0f22de90b9 100644 --- a/ui-ngx/src/app/shared/models/constants.ts +++ b/ui-ngx/src/app/shared/models/constants.ts @@ -194,6 +194,7 @@ export const HelpLinks = { scada: `${helpBaseUrl}/docs${docPlatformPrefix}/user-guide/scada`, scadaSymbolDev: `${helpBaseUrl}/docs${docPlatformPrefix}/user-guide/scada/scada-symbols-dev-guide/`, scadaSymbolDevAnimation: `${helpBaseUrl}/docs${docPlatformPrefix}/user-guide/scada/scada-symbols-dev-guide/#scadasymbolanimation`, + scadaSymbolDevConnectorAnimation: `${helpBaseUrl}/docs${docPlatformPrefix}/user-guide/scada/scada-symbols-dev-guide/#connectorscadasymbolanimation`, domains: `${helpBaseUrl}/docs${docPlatformPrefix}/domains`, mobileApplication: `${helpBaseUrl}/docs${docPlatformPrefix}/mobile-center/applications/`, mobileBundle: `${helpBaseUrl}/docs${docPlatformPrefix}/mobile-center/mobile-center/`, From 71724b95d6a0e903c24df7148f02bddde423bd76 Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Fri, 2 May 2025 13:14:51 +0300 Subject: [PATCH 20/79] UI: Fixed percent value for doughnut chart --- .../home/components/widget/lib/chart/pie-chart.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/pie-chart.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/pie-chart.ts index 2cb5c5a031..c985fcb55a 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/pie-chart.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/pie-chart.ts @@ -122,17 +122,10 @@ export class TbPieChart extends TbLatestChart { seriesData.push( {id: dataItem.id, value: dataItem.value, name: dataItem.dataKey.label, itemStyle: {color: dataItem.dataKey.color}} ); - if (this.settings.doughnut && enabledDataItems.length > 1) { - seriesData.push({ - value: 0, name: '', itemStyle: {color: 'transparent'}, emphasis: {disabled: true} - }); - } } } if (this.settings.doughnut) { - for (let i = 1; i < seriesData.length; i += 2) { - seriesData[i].value = this.total / 100; - } + this.latestChartOption.series[0].padAngle = 2; } this.latestChartOption.series[0].data = seriesData; } From 1c5fc16b58eb37d82d08b3fc55f853e3cfc47f86 Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Fri, 2 May 2025 13:19:40 +0300 Subject: [PATCH 21/79] UI: Remove unused constant --- .../app/modules/home/components/widget/lib/chart/pie-chart.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/pie-chart.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/pie-chart.ts index c985fcb55a..ec56ce5478 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/pie-chart.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/pie-chart.ts @@ -116,7 +116,6 @@ export class TbPieChart extends TbLatestChart { protected doUpdateSeriesData() { const seriesData: PieDataItemOption[] = []; - const enabledDataItems = this.dataItems.filter(item => item.enabled && item.hasValue); for (const dataItem of this.dataItems) { if (dataItem.enabled && dataItem.hasValue) { seriesData.push( From 23ded89dea5498c626b84cc0d4637a14e11c71ed Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Fri, 2 May 2025 13:31:37 +0300 Subject: [PATCH 22/79] UI: Refactoring --- .../app/modules/home/components/widget/lib/chart/pie-chart.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/pie-chart.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/pie-chart.ts index ec56ce5478..d4279caa69 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/pie-chart.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/pie-chart.ts @@ -116,6 +116,7 @@ export class TbPieChart extends TbLatestChart { protected doUpdateSeriesData() { const seriesData: PieDataItemOption[] = []; + const enabledDataItems = this.dataItems.filter(item => item.enabled && item.hasValue); for (const dataItem of this.dataItems) { if (dataItem.enabled && dataItem.hasValue) { seriesData.push( @@ -124,7 +125,7 @@ export class TbPieChart extends TbLatestChart { } } if (this.settings.doughnut) { - this.latestChartOption.series[0].padAngle = 2; + this.latestChartOption.series[0].padAngle = enabledDataItems.length > 1 ? 2 : 0; } this.latestChartOption.series[0].data = seriesData; } From 8dcd8f707d9a1686caa6c6b47754113a0026d5c3 Mon Sep 17 00:00:00 2001 From: Yevhenii Date: Tue, 6 May 2025 09:37:21 +0300 Subject: [PATCH 23/79] Update edge install instructions - changed instructions --- .../edge/instructions/install/docker/instructions.md | 10 ++++++---- .../edge/instructions/install/ubuntu/instructions.md | 10 +++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/application/src/main/data/json/edge/instructions/install/docker/instructions.md b/application/src/main/data/json/edge/instructions/install/docker/instructions.md index 4457962d86..3d42cffa8a 100644 --- a/application/src/main/data/json/edge/instructions/install/docker/instructions.md +++ b/application/src/main/data/json/edge/instructions/install/docker/instructions.md @@ -37,16 +37,18 @@ version: '3.8' services: mytbedge: restart: always - image: "thingsboard/tb-edge:3.9.1EDGE" + image: "thingsboard/tb-edge:${TB_EDGE_VERSION}" ports: - "8080:8080" - "1883:1883" - "5683-5688:5683-5688/udp" environment: SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/tb-edge - CLOUD_ROUTING_KEY: PUT_YOUR_EDGE_KEY_HERE # e.g. 19ea7ee8-5e6d-e642-4f32-05440a529015 - CLOUD_ROUTING_SECRET: PUT_YOUR_EDGE_SECRET_HERE # e.g. bztvkvfqsye7omv9uxlp - CLOUD_RPC_HOST: PUT_YOUR_CLOUD_IP # e.g. 192.168.1.1 or demo.thingsboard.io + CLOUD_ROUTING_KEY: ${CLOUD_ROUTING_KEY} + CLOUD_ROUTING_SECRET: ${CLOUD_ROUTING_SECRET} + CLOUD_RPC_HOST: ${BASE_URL} + CLOUD_RPC_PORT: ${CLOUD_RPC_PORT} + CLOUD_RPC_SSL_ENABLED: ${CLOUD_RPC_SSL_ENABLED} volumes: - tb-edge-data:/data - tb-edge-logs:/var/log/tb-edge diff --git a/application/src/main/data/json/edge/instructions/install/ubuntu/instructions.md b/application/src/main/data/json/edge/instructions/install/ubuntu/instructions.md index 9d685c576a..6ac45d325a 100644 --- a/application/src/main/data/json/edge/instructions/install/ubuntu/instructions.md +++ b/application/src/main/data/json/edge/instructions/install/ubuntu/instructions.md @@ -99,11 +99,11 @@ To configure ThingsBoard Edge, you can use the following command to automatical ```bash sudo sh -c 'cat <> /etc/tb-edge/conf/tb-edge.conf -export CLOUD_ROUTING_KEY= -export CLOUD_ROUTING_SECRET= -export CLOUD_RPC_HOST=demo.thingsboard.io -export CLOUD_RPC_PORT=7070 -export CLOUD_RPC_SSL_ENABLED=false +export CLOUD_ROUTING_KEY=${CLOUD_ROUTING_KEY} +export CLOUD_ROUTING_SECRET=${CLOUD_ROUTING_SECRET} +export CLOUD_RPC_HOST=${BASE_URL} +export CLOUD_RPC_PORT=${CLOUD_RPC_PORT} +export CLOUD_RPC_SSL_ENABLED=${CLOUD_RPC_SSL_ENABLED} EOL' {:copy-code} ``` From a0a77c721f5081e8bbfab4eed6321942290a7f60 Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Tue, 6 May 2025 13:31:34 +0200 Subject: [PATCH 24/79] fixed 'value too long' after saving ota (#13277) * fixed 'value too long' after saving ota * fixed tests --- .../server/service/entitiy/ota/DefaultTbOtaPackageService.java | 2 +- .../thingsboard/server/controller/OtaPackageControllerTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java index fca015671e..af8bbeb669 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java @@ -86,7 +86,7 @@ public class DefaultTbOtaPackageService extends AbstractTbEntityService implemen otaPackage.setContentType(contentType); otaPackage.setData(ByteBuffer.wrap(data)); otaPackage.setDataSize((long) data.length); - OtaPackageInfo savedOtaPackage = otaPackageService.saveOtaPackage(otaPackage); + OtaPackageInfo savedOtaPackage = new OtaPackageInfo(otaPackageService.saveOtaPackage(otaPackage)); logEntityActionService.logEntityAction(tenantId, savedOtaPackage.getId(), savedOtaPackage, null, actionType, user); return savedOtaPackage; } catch (Exception e) { diff --git a/application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java index e8bb65dd49..3fc839996c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java @@ -216,7 +216,7 @@ public class OtaPackageControllerTest extends AbstractControllerTest { Assert.assertEquals(CHECKSUM_ALGORITHM, savedFirmware.getChecksumAlgorithm().name()); Assert.assertEquals(CHECKSUM, savedFirmware.getChecksum()); - testNotifyEntityAllOneTime(savedFirmware, savedFirmware.getId(), savedFirmware.getId(), + testNotifyEntityAllOneTime(new OtaPackageInfo(savedFirmware), savedFirmware.getId(), savedFirmware.getId(), savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.UPDATED); } From 91c5a0514051dcbfc9dfb63e15860e373e14bb78 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 7 May 2025 10:15:56 +0300 Subject: [PATCH 25/79] fixed telemetry updates with old timestamps --- .../DefaultTbLocalSubscriptionService.java | 4 +- .../subscription/TbEntityDataSubCtx.java | 49 +++---- ...DefaultTbLocalSubscriptionServiceTest.java | 73 ++++++++-- .../subscription/TbEntityDataSubCtxTest.java | 130 ++++++++++++++++++ 4 files changed, 216 insertions(+), 40 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/service/subscription/TbEntityDataSubCtxTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java index 5716976b6f..eed68b0c5e 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java @@ -348,7 +348,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer if (sub.isLatestValues()) { for (TsKvEntry kv : data) { Long stateTs = keyStates.get(kv.getKey()); - if (stateTs == null || kv.getTs() > stateTs) { + if (stateTs == null || kv.getTs() >= stateTs) { if (updateData == null) { updateData = new ArrayList<>(); } @@ -362,7 +362,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer for (TsKvEntry kv : data) { Long stateTs = keyStates.get(kv.getKey()); if (stateTs != null) { - if (!sub.isLatestValues() || kv.getTs() > stateTs) { + if (!sub.isLatestValues() || kv.getTs() >= stateTs) { if (updateData == null) { updateData = new ArrayList<>(); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbEntityDataSubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbEntityDataSubCtx.java index d397bbc05b..883b5307d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbEntityDataSubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbEntityDataSubCtx.java @@ -94,35 +94,36 @@ public class TbEntityDataSubCtx extends TbAbstractDataSubCtx { private void sendLatestWsMsg(EntityId entityId, String sessionId, TelemetrySubscriptionUpdate subscriptionUpdate, EntityKeyType keyType) { Map latestUpdate = new HashMap<>(); - subscriptionUpdate.getData().forEach((k, v) -> { - Object[] data = (Object[]) v.get(0); - latestUpdate.put(k, new TsValue((Long) data[0], (String) data[1])); - }); + subscriptionUpdate.getData().forEach((key, data) -> data.stream() + .filter(o -> o instanceof Object[] && ((Object[]) o).length > 0 && ((Object[]) o)[0] instanceof Long) + .max(Comparator.comparingLong(o -> (Long) ((Object[]) o)[0])) + .ifPresent(max -> { + Object[] arr = (Object[]) max; + latestUpdate.put(key, new TsValue((Long) arr[0], (String) arr[1])); + })); EntityData entityData = getDataForEntity(entityId); if (entityData != null && entityData.getLatest() != null) { - Map latestCtxValues = entityData.getLatest().get(keyType); + Map latestCtxValues = entityData.getLatest().computeIfAbsent(keyType, key -> new HashMap<>()); log.trace("[{}][{}][{}] Going to compare update with {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), latestCtxValues); - if (latestCtxValues != null) { - latestCtxValues.forEach((k, v) -> { - TsValue update = latestUpdate.get(k); - if (update != null) { - //Ignore notifications about deleted keys - if (!(update.getTs() == 0 && (update.getValue() == null || update.getValue().isEmpty()))) { - if (update.getTs() < v.getTs()) { - log.trace("[{}][{}][{}] Removed stale update for key: {} and ts: {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), k, update.getTs()); - latestUpdate.remove(k); - } else if ((update.getTs() == v.getTs() && update.getValue().equals(v.getValue()))) { - log.trace("[{}][{}][{}] Removed duplicate update for key: {} and ts: {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), k, update.getTs()); - latestUpdate.remove(k); - } - } else { - log.trace("[{}][{}][{}] Received deleted notification for: {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), k); + latestCtxValues.forEach((k, v) -> { + TsValue update = latestUpdate.get(k); + if (update != null) { + //Ignore notifications about deleted keys + if (!(update.getTs() == 0 && (update.getValue() == null || update.getValue().isEmpty()))) { + if (update.getTs() < v.getTs()) { + log.trace("[{}][{}][{}] Removed stale update for key: {} and ts: {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), k, update.getTs()); + latestUpdate.remove(k); + } else if ((update.getTs() == v.getTs() && update.getValue().equals(v.getValue()))) { + log.trace("[{}][{}][{}] Removed duplicate update for key: {} and ts: {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), k, update.getTs()); + latestUpdate.remove(k); } + } else { + log.trace("[{}][{}][{}] Received deleted notification for: {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), k); } - }); - //Setting new values - latestUpdate.forEach(latestCtxValues::put); - } + } + }); + //Setting new values + latestCtxValues.putAll(latestUpdate); } if (!latestUpdate.isEmpty()) { Map> latestMap = Collections.singletonMap(keyType, latestUpdate); diff --git a/application/src/test/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionServiceTest.java index 2630f95f9b..623dd6b9e2 100644 --- a/application/src/test/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionServiceTest.java @@ -32,11 +32,16 @@ import org.thingsboard.server.cache.limits.RateLimitService; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.limit.LimitedApi; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.service.ws.WebSocketSessionRef; +import org.thingsboard.server.service.ws.telemetry.sub.TelemetrySubscriptionUpdate; import java.util.ArrayList; import java.util.HashMap; @@ -44,8 +49,11 @@ import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.nullable; @@ -54,11 +62,17 @@ import static org.mockito.Mockito.when; public class DefaultTbLocalSubscriptionServiceTest { - ListAppender testLogAppender; - TbLocalSubscriptionService subscriptionService; + private final TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); + private final DeviceId deviceId = new DeviceId(UUID.randomUUID()); + + private ListAppender testLogAppender; + private TbLocalSubscriptionService subscriptionService; + private ListeningExecutorService executorService; @BeforeEach public void setUp() throws Exception { + this.executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(10)); + Logger logger = (Logger) LoggerFactory.getLogger(DefaultTbLocalSubscriptionService.class); testLogAppender = new ListAppender<>(); testLogAppender.start(); @@ -69,7 +83,9 @@ public class DefaultTbLocalSubscriptionServiceTest { PartitionService partitionService = mock(); when(partitionService.resolve(any(), any(), any())).thenReturn(TopicPartitionInfo.builder().build()); subscriptionService = new DefaultTbLocalSubscriptionService(mock(), mock(), mock(), partitionService, mock(), mock(), mock(), rateLimitService); + ReflectionTestUtils.setField(subscriptionService, "serviceId", "serviceId"); + ReflectionTestUtils.setField(subscriptionService, "subscriptionUpdateExecutor", executorService); } @AfterEach @@ -79,27 +95,22 @@ public class DefaultTbLocalSubscriptionServiceTest { Logger logger = (Logger) LoggerFactory.getLogger(DefaultTbLocalSubscriptionService.class); logger.detachAppender(testLogAppender); } + if (executorService != null) { + executorService.shutdownNow(); + } } @Test public void addSubscriptionConcurrentModificationTest() throws Exception { - ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(10)); - TenantId tenantId = new TenantId(UUID.randomUUID()); - DeviceId deviceId = new DeviceId(UUID.randomUUID()); WebSocketSessionRef sessionRef = mock(); - ReflectionTestUtils.setField(subscriptionService, "subscriptionUpdateExecutor", executorService); List> futures = new ArrayList<>(); - try { - subscriptionService.onCoreStartupMsg(TransportProtos.CoreStartupMsg.newBuilder().addAllPartitions(List.of(0)).getDefaultInstanceForType()); - for (int i = 0; i < 50; i++) { - futures.add(executorService.submit(() -> subscriptionService.addSubscription(createSubscription(tenantId, deviceId), sessionRef))); - } - Futures.allAsList(futures).get(); - } finally { - executorService.shutdownNow(); + subscriptionService.onCoreStartupMsg(TransportProtos.CoreStartupMsg.newBuilder().addAllPartitions(List.of(0)).getDefaultInstanceForType()); + for (int i = 0; i < 50; i++) { + futures.add(executorService.submit(() -> subscriptionService.addSubscription(createSubscription(tenantId, deviceId), sessionRef))); } + Futures.allAsList(futures).get(); List logs = testLogAppender.list; boolean exceptionLogged = logs.stream() @@ -123,4 +134,38 @@ public class DefaultTbLocalSubscriptionServiceTest { .keyStates(keys) .build(); } + + @Test + public void updateOldTelemetryTest() { + WebSocketSessionRef sessionRef = mock(); + + String key = "temperature"; + long ts = 1237465456L; + + Map keyStates = new HashMap<>(); + keyStates.put(key, ts); + + AtomicReference capturedUpdate = new AtomicReference<>(); + TbTimeSeriesSubscription tsSubscription = TbTimeSeriesSubscription.builder() + .tenantId(tenantId) + .entityId(deviceId) + .subscriptionId(2) + .sessionId(RandomStringUtils.randomAlphanumeric(5)) + .keyStates(keyStates) + .allKeys(true) + .latestValues(true) + .updateProcessor((subscription, update) -> capturedUpdate.set(update)) + .build(); + + subscriptionService.addSubscription(tsSubscription, sessionRef); + + // Send telemetry with ts == stateTs + TsKvEntry kv = new BasicTsKvEntry(ts, new StringDataEntry(key, "42")); + subscriptionService.onTimeSeriesUpdate(deviceId, List.of(kv), mock(TbCallback.class)); + + TelemetrySubscriptionUpdate update = capturedUpdate.get(); + assertNotNull(update); + assertTrue(update.getLatestValues().containsKey(key)); + } + } diff --git a/application/src/test/java/org/thingsboard/server/service/subscription/TbEntityDataSubCtxTest.java b/application/src/test/java/org/thingsboard/server/service/subscription/TbEntityDataSubCtxTest.java new file mode 100644 index 0000000000..1f0da1a30c --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/subscription/TbEntityDataSubCtxTest.java @@ -0,0 +1,130 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.subscription; + +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomUtils; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.TsValue; +import org.thingsboard.server.service.ws.WebSocketService; +import org.thingsboard.server.service.ws.WebSocketSessionRef; +import org.thingsboard.server.service.ws.telemetry.cmd.v2.CmdUpdate; +import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityDataUpdate; +import org.thingsboard.server.service.ws.telemetry.sub.TelemetrySubscriptionUpdate; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.then; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class TbEntityDataSubCtxTest { + + private final DeviceId deviceId = new DeviceId(UUID.randomUUID()); + + private final Integer cmdId = RandomUtils.nextInt(); + private final Integer subscriptionId = RandomUtils.nextInt(); + private final String serviceId = RandomStringUtils.randomAlphanumeric(10); + private final String sessionId = RandomStringUtils.randomAlphanumeric(10); + + private final int maxEntitiesPerDataSubscription = 100; + + private TbEntityDataSubCtx subCtx; + @Mock + private WebSocketService webSocketService; + @Mock + private WebSocketSessionRef webSocketSessionRef; + + @BeforeEach + public void setUp() { + when(webSocketSessionRef.getSessionId()).thenReturn(sessionId); + subCtx = new TbEntityDataSubCtx(serviceId, webSocketService, mock(), mock(), mock(), mock(), webSocketSessionRef, cmdId, maxEntitiesPerDataSubscription); + + Map subToEntityIdMap = new HashMap<>(); + subToEntityIdMap.put(subscriptionId, deviceId); + ReflectionTestUtils.setField(subCtx, "subToEntityIdMap", subToEntityIdMap); + + long now = System.currentTimeMillis(); + long oldTs = now - 1_000_000; + + Map latestCtxValues = new HashMap<>(); + latestCtxValues.put("key", new TsValue(oldTs, "15")); + Map> latest = new HashMap<>(); + latest.put(EntityKeyType.TIME_SERIES, latestCtxValues); + + EntityData entityData = new EntityData(); + entityData.setEntityId(deviceId); + entityData.setLatest(latest); + + PageData data = new PageData<>(List.of(entityData), 1, 1, true); + ReflectionTestUtils.setField(subCtx, "data", data); + } + + @Test + public void testSendLatestWsMsg() { + long ts = System.currentTimeMillis(); + List telemetry = List.of( + new BasicTsKvEntry(ts - 50000, new LongDataEntry("key", 42L), 34L), + new BasicTsKvEntry(ts - 20000, new LongDataEntry("key", 17L), 78L) + ); + + TelemetrySubscriptionUpdate subUpdate = new TelemetrySubscriptionUpdate(subscriptionId, telemetry); + + subCtx.sendWsMsg(sessionId, subUpdate, EntityKeyType.TIME_SERIES, true); + + Map> expectedLatest = new HashMap<>(); + Map expectedLatestCtxValues = new HashMap<>(); + expectedLatestCtxValues.put("key", new TsValue(ts - 20000, "17")); // use latest telemetry + expectedLatest.put(EntityKeyType.TIME_SERIES, expectedLatestCtxValues); + + EntityData expectedEntityData = new EntityData(); + expectedEntityData.setEntityId(deviceId); + expectedEntityData.setLatest(expectedLatest); + + List expected = List.of(expectedEntityData); + + ArgumentCaptor cmdUpdateCaptor = ArgumentCaptor.forClass(CmdUpdate.class); + then(webSocketService).should().sendUpdate(eq(sessionId), cmdUpdateCaptor.capture()); + CmdUpdate cmdUpdate = cmdUpdateCaptor.getValue(); + assertThat(cmdUpdate).isInstanceOf(EntityDataUpdate.class); + EntityDataUpdate entityDataUpdate = (EntityDataUpdate) cmdUpdate; + assertThat(entityDataUpdate.getCmdId()).isEqualTo(cmdId); + assertThat(entityDataUpdate.getData()).isNull(); + assertThat(entityDataUpdate.getUpdate()).isEqualTo(expected); + assertThat(entityDataUpdate.getAllowedEntities()).isEqualTo(maxEntitiesPerDataSubscription); + } + +} From c8b13fc3507645fb6d690d7b131708067b0b9891 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 7 May 2025 11:13:36 +0300 Subject: [PATCH 26/79] added await for test --- .../DefaultTbLocalSubscriptionServiceTest.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionServiceTest.java index 623dd6b9e2..3bb470830b 100644 --- a/application/src/test/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionServiceTest.java @@ -49,8 +49,10 @@ import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -163,9 +165,13 @@ public class DefaultTbLocalSubscriptionServiceTest { TsKvEntry kv = new BasicTsKvEntry(ts, new StringDataEntry(key, "42")); subscriptionService.onTimeSeriesUpdate(deviceId, List.of(kv), mock(TbCallback.class)); - TelemetrySubscriptionUpdate update = capturedUpdate.get(); - assertNotNull(update); - assertTrue(update.getLatestValues().containsKey(key)); + await().atMost(5, TimeUnit.SECONDS) + .pollInterval(5, TimeUnit.MILLISECONDS) + .untilAsserted(() -> { + TelemetrySubscriptionUpdate update = capturedUpdate.get(); + assertNotNull(update); + assertTrue(update.getLatestValues().containsKey(key)); + }); } } From dd76f2d8231cc754e9254bce68b63d13e17a2674 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 7 May 2025 11:44:54 +0300 Subject: [PATCH 27/79] UI: Fixed rule node config style --- .../gps-geo-action-config.component.html | 2 +- .../math-function-config.component.html | 2 +- ...save-to-custom-table-config.component.html | 2 +- .../arguments-map-config.component.html | 2 +- ...vice-relations-query-config.component.html | 2 +- .../message-types-config.component.html | 2 +- .../common/select-attributes.component.html | 2 +- .../entity-details-config.component.html | 2 +- .../external/lambda-config.component.html | 4 +- .../check-message-config.component.html | 4 +- .../originator-type-config.component.html | 2 +- .../copy-keys-config.component.html | 2 +- .../deduplication-config.component.html | 6 +-- .../delete-keys-config.component.html | 2 +- .../rulechain/rule-node-config.component.html | 2 +- .../rulechain/rule-node-config.component.scss | 53 ++++++++++++++++++- .../rulechain/rule-node-config.component.ts | 10 ++-- 17 files changed, 77 insertions(+), 24 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/gps-geo-action-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/gps-geo-action-config.component.html index 37e93157d3..2d3f230643 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/gps-geo-action-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/gps-geo-action-config.component.html @@ -116,7 +116,7 @@ rule-node-config.polygon-definition - help diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/math-function-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/math-function-config.component.html index 3147162f06..4d4940404e 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/math-function-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/math-function-config.component.html @@ -77,7 +77,7 @@ rule-node-config.key-field-input - help diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/save-to-custom-table-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/save-to-custom-table-config.component.html index c1d6dd3215..c9685f2c3f 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/save-to-custom-table-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/save-to-custom-table-config.component.html @@ -19,7 +19,7 @@ rule-node-config.custom-table-name - rule-node-config.argument-key-field-input - help diff --git a/ui-ngx/src/app/modules/home/components/rule-node/common/device-relations-query-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/common/device-relations-query-config.component.html index ce99d6f7e6..0da5cae858 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/common/device-relations-query-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/common/device-relations-query-config.component.html @@ -60,7 +60,7 @@ [emptyInputPlaceholder]="'rule-node-config.add-device-profile' | translate" [filledInputPlaceholder]="'rule-node-config.add-device-profile' | translate" formControlName="deviceTypes"> - help diff --git a/ui-ngx/src/app/modules/home/components/rule-node/common/message-types-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/common/message-types-config.component.html index ab436088bd..dcc968dee3 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/common/message-types-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/common/message-types-config.component.html @@ -62,7 +62,7 @@ - help diff --git a/ui-ngx/src/app/modules/home/components/rule-node/common/select-attributes.component.html b/ui-ngx/src/app/modules/home/components/rule-node/common/select-attributes.component.html index 88c4cc1bdb..49669c147f 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/common/select-attributes.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/common/select-attributes.component.html @@ -53,7 +53,7 @@ - help diff --git a/ui-ngx/src/app/modules/home/components/rule-node/enrichment/entity-details-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/enrichment/entity-details-config.component.html index 11f0323346..5d80cfb276 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/enrichment/entity-details-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/enrichment/entity-details-config.component.html @@ -22,7 +22,7 @@ [placeholder]="'rule-node-config.add-detail' | translate" [requiredText]="'rule-node-config.entity-details-list-empty' | translate" formControlName="detailsList"> - help diff --git a/ui-ngx/src/app/modules/home/components/rule-node/external/lambda-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/external/lambda-config.component.html index 67f0aaca7f..7de1a46a9d 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/external/lambda-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/external/lambda-config.component.html @@ -85,7 +85,7 @@ {{ 'rule-node-config.connection-timeout-min' | translate }} - help @@ -98,7 +98,7 @@ {{ 'rule-node-config.request-timeout-min' | translate }} - help diff --git a/ui-ngx/src/app/modules/home/components/rule-node/filter/check-message-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/filter/check-message-config.component.html index 7dd89d4927..0bdba52c1e 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/filter/check-message-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/filter/check-message-config.component.html @@ -26,14 +26,14 @@ [label]="'rule-node-config.data-keys' | translate" [placeholder]="'rule-node-config.add-message-field' | translate" formControlName="messageNames"> - help - help
- help diff --git a/ui-ngx/src/app/modules/home/components/rule-node/transformation/copy-keys-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/transformation/copy-keys-config.component.html index b1e810bcb2..3d524b8402 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/transformation/copy-keys-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/transformation/copy-keys-config.component.html @@ -27,7 +27,7 @@ [placeholder]="'rule-node-config.add-key' | translate" [requiredText]="'rule-node-config.key-val.at-least-one-key-error' | translate" formControlName="keys"> - help diff --git a/ui-ngx/src/app/modules/home/components/rule-node/transformation/deduplication-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/transformation/deduplication-config.component.html index 3df9c1d7d3..aa6ff0bab7 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/transformation/deduplication-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/transformation/deduplication-config.component.html @@ -25,7 +25,7 @@ {{'rule-node-config.interval-min-error' | translate}} - help @@ -73,7 +73,7 @@ {{'rule-node-config.max-pending-msgs-min-error' | translate}} - help @@ -89,7 +89,7 @@ {{'rule-node-config.max-retries-min-error' | translate}} - help diff --git a/ui-ngx/src/app/modules/home/components/rule-node/transformation/delete-keys-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/transformation/delete-keys-config.component.html index 75f9c243e9..4385341686 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/transformation/delete-keys-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/transformation/delete-keys-config.component.html @@ -26,7 +26,7 @@ [placeholder]="'rule-node-config.add-key' | translate" [requiredText]="'rule-node-config.key-val.at-least-one-key-error' | translate" formControlName="keys"> - help diff --git a/ui-ngx/src/app/modules/home/pages/rulechain/rule-node-config.component.html b/ui-ngx/src/app/modules/home/pages/rulechain/rule-node-config.component.html index 3522c584f4..0228b8d973 100644 --- a/ui-ngx/src/app/modules/home/pages/rulechain/rule-node-config.component.html +++ b/ui-ngx/src/app/modules/home/pages/rulechain/rule-node-config.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+
{{definedDirectiveError}}
* { + flex: 1; + } + + .flex-2 { + flex: 2; + } + + .third-width { + max-width: 32%; + @media #{$mat-xs} { + max-width: 100%; + } + } + } } diff --git a/ui-ngx/src/app/modules/home/pages/rulechain/rule-node-config.component.ts b/ui-ngx/src/app/modules/home/pages/rulechain/rule-node-config.component.ts index e3e5d939df..d27e4d8abb 100644 --- a/ui-ngx/src/app/modules/home/pages/rulechain/rule-node-config.component.ts +++ b/ui-ngx/src/app/modules/home/pages/rulechain/rule-node-config.component.ts @@ -19,11 +19,13 @@ import { ComponentRef, EventEmitter, forwardRef, + HostBinding, Input, OnDestroy, Output, ViewChild, - ViewContainerRef + ViewContainerRef, + ViewEncapsulation } from '@angular/core'; import { ControlValueAccessor, @@ -52,14 +54,16 @@ import { RuleChainType } from '@shared/models/rule-chain.models'; provide: NG_VALUE_ACCESSOR, useExisting: forwardRef(() => RuleNodeConfigComponent), multi: true - }] + }], + encapsulation: ViewEncapsulation.None }) export class RuleNodeConfigComponent implements ControlValueAccessor, OnDestroy { @ViewChild('definedConfigContent', {read: ViewContainerRef, static: true}) definedConfigContainer: ViewContainerRef; - @ViewChild('jsonObjectEditComponent') jsonObjectEditComponent: JsonObjectEditComponent; + @HostBinding('style.display') readonly styleDisplay = 'block'; + private requiredValue: boolean; get required(): boolean { return this.requiredValue; From 3ea08293fa4982ce24660a07d8878af4fd777c1d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 7 May 2025 16:32:25 +0300 Subject: [PATCH 28/79] removed resource CF --- .../org/thingsboard/server/controller/BaseController.java | 8 ++++++-- .../server/controller/CalculatedFieldController.java | 6 ++---- .../server/service/security/permission/Resource.java | 3 +-- .../security/permission/TenantAdminPermissions.java | 1 - 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 73e278389a..d29d398dca 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -964,8 +964,12 @@ public abstract class BaseController { } } - protected CalculatedField checkCalculatedFieldId(CalculatedFieldId calculatedFieldId, Operation operation) throws ThingsboardException { - return checkEntityId(calculatedFieldId, calculatedFieldService::findById, operation); + private void checkCalculatedFieldId(CalculatedFieldId calculatedFieldId, Operation operation) throws ThingsboardException { + validateId(calculatedFieldId, "Invalid entity id"); + SecurityUser user = getCurrentUser(); + CalculatedField cf = calculatedFieldService.findById(user.getTenantId(), calculatedFieldId); + checkNotNull(cf, calculatedFieldId.getEntityType().getNormalName() + " with id [" + calculatedFieldId + "] is not found"); + checkEntityId(cf.getEntityId(), operation); } protected HomeDashboardInfo getHomeDashboardInfo(SecurityUser securityUser, JsonNode additionalInfo) { diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index f899d0f480..1c00988f6a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -59,7 +59,6 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldTbelScriptEngi import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.security.permission.Resource; import java.util.ArrayList; import java.util.Collections; @@ -136,7 +135,6 @@ public class CalculatedFieldController extends BaseController { public CalculatedField saveCalculatedField(@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "A JSON value representing the calculated field.") @RequestBody CalculatedField calculatedField) throws Exception { calculatedField.setTenantId(getTenantId()); - checkEntity(calculatedField.getId(), calculatedField, Resource.CALCULATED_FIELD); checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); checkReferencedEntities(calculatedField.getConfiguration(), getCurrentUser()); return tbCalculatedFieldService.save(calculatedField, getCurrentUser()); @@ -186,7 +184,7 @@ public class CalculatedFieldController extends BaseController { public void deleteCalculatedField(@PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws Exception { checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); - CalculatedField calculatedField = checkCalculatedFieldId(calculatedFieldId, Operation.DELETE); + CalculatedField calculatedField = tbCalculatedFieldService.findById(calculatedFieldId, getCurrentUser()); checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); tbCalculatedFieldService.delete(calculatedField, getCurrentUser()); } @@ -200,7 +198,7 @@ public class CalculatedFieldController extends BaseController { public JsonNode getLatestCalculatedFieldDebugEvent(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); - CalculatedField calculatedField = checkCalculatedFieldId(calculatedFieldId, Operation.READ); + CalculatedField calculatedField = tbCalculatedFieldService.findById(calculatedFieldId, getCurrentUser()); checkEntityId(calculatedField.getEntityId(), Operation.READ_CALCULATED_FIELD); TenantId tenantId = getCurrentUser().getTenantId(); return Optional.ofNullable(eventService.findLatestEvents(tenantId, calculatedFieldId, EventType.DEBUG_CALCULATED_FIELD, 1)) diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java index 9d7590f786..4cb281a719 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java @@ -50,8 +50,7 @@ public enum Resource { VERSION_CONTROL, NOTIFICATION(EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_RULE), - MOBILE_APP_SETTINGS, - CALCULATED_FIELD(EntityType.CALCULATED_FIELD); + MOBILE_APP_SETTINGS; private final Set entityTypes; diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index a072cf2738..7a67d6739e 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -55,7 +55,6 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.OAUTH2_CONFIGURATION_TEMPLATE, new PermissionChecker.GenericPermissionChecker(Operation.READ)); put(Resource.MOBILE_APP, tenantEntityPermissionChecker); put(Resource.MOBILE_APP_BUNDLE, tenantEntityPermissionChecker); - put(Resource.CALCULATED_FIELD, tenantEntityPermissionChecker); } public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { From 81e35e134e8377f2e24f106aba1188a235aee153 Mon Sep 17 00:00:00 2001 From: Yevhenii Date: Wed, 7 May 2025 17:40:59 +0300 Subject: [PATCH 29/79] Validation DefaultEdgeRuleChain in DeviceProfile - added validation for DefaultEdgeRuleChain - refactoring --- .../validator/DeviceProfileDataValidator.java | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java index a9e14229f8..e32195022c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java @@ -43,6 +43,7 @@ import org.thingsboard.server.common.data.device.profile.lwm2m.bootstrap.Abstrac import org.thingsboard.server.common.data.device.profile.lwm2m.bootstrap.LwM2MBootstrapServerCredential; import org.thingsboard.server.common.data.device.profile.lwm2m.bootstrap.RPKLwM2MBootstrapServerCredential; import org.thingsboard.server.common.data.device.profile.lwm2m.bootstrap.X509LwM2MBootstrapServerCredential; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.rule.RuleChain; @@ -188,13 +189,11 @@ public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator 65535){ + if (serverConfig.isBootstrapServerIs()) { + if (serverConfig.getShortServerId() < 0 || serverConfig.getShortServerId() > 65535) { throw new DeviceCredentialsValidationException("Bootstrap Server ShortServerId must be in range [0 - 65535]!"); } } else { @@ -423,4 +432,5 @@ public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator Date: Fri, 9 May 2025 10:08:58 +0300 Subject: [PATCH 30/79] UI: Remove sticky option for argument table header row --- .../calculated-field-arguments-table.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index 53d8a9d7b6..b4b5a939e1 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -116,7 +116,7 @@ + *matHeaderRowDef="['name', 'entityType', 'target', 'type', 'key', 'actions']"> From ecc82d8c048ed7f0d699ff54dc1d53c0ac623d5d Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Fri, 9 May 2025 12:23:51 +0300 Subject: [PATCH 31/79] UI: Event table hide clear event button as input --- .../app/modules/home/components/event/event-table-config.ts | 4 ++-- .../modules/home/components/event/event-table.component.ts | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 0335c43f29..03fc641735 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -59,7 +59,6 @@ import { AppState } from '@core/core.state'; export class EventTableConfig extends EntityTableConfig { eventTypeValue: EventType | DebugEventType; - hideClearEventAction = false; private filterParams: FilterEventBody = {}; private filterColumns: FilterEntityColumn[] = []; @@ -95,7 +94,8 @@ export class EventTableConfig extends EntityTableConfig { private cd: ChangeDetectorRef, private store: Store, public testButtonLabel?: string, - private debugEventSelected?: EventEmitter) { + private debugEventSelected?: EventEmitter, + public hideClearEventAction = false) { super(); this.loadDataOnInit = false; this.tableTitle = ''; diff --git a/ui-ngx/src/app/modules/home/components/event/event-table.component.ts b/ui-ngx/src/app/modules/home/components/event/event-table.component.ts index d12791680b..f51cf66d8a 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table.component.ts @@ -58,6 +58,9 @@ export class EventTableComponent implements OnInit, AfterViewInit, OnDestroy { @Input() debugEventTypes: Array; + @Input() + hideClearEventAction: boolean = false; + activeValue = false; dirtyValue = false; entityIdValue: EntityId; @@ -147,7 +150,8 @@ export class EventTableComponent implements OnInit, AfterViewInit, OnDestroy { this.cd, this.store, this.functionTestButtonLabel, - this.debugEventSelected + this.debugEventSelected, + this.hideClearEventAction ); } From 590805b6a3ec92e868e6a9b6427cc32f4670a998 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 9 May 2025 13:25:31 +0300 Subject: [PATCH 32/79] Handle uncaught auth exceptions --- .../ThingsboardSecurityConfiguration.java | 7 ++- .../server/controller/BaseController.java | 9 +--- .../ThingsboardErrorResponseHandler.java | 26 ++++++++++- .../security/auth/AuthExceptionHandler.java | 46 +++++++++++++++++++ .../data/exception/ThingsboardErrorCode.java | 3 +- 5 files changed, 80 insertions(+), 11 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/security/auth/AuthExceptionHandler.java diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java index 871798ccd1..93cf9da8e0 100644 --- a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java @@ -47,6 +47,7 @@ import org.springframework.web.filter.ShallowEtagHeaderFilter; import org.thingsboard.server.dao.oauth2.OAuth2Configuration; import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.auth.AuthExceptionHandler; import org.thingsboard.server.service.security.auth.jwt.JwtAuthenticationProvider; import org.thingsboard.server.service.security.auth.jwt.JwtTokenAuthenticationProcessingFilter; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenAuthenticationProvider; @@ -129,6 +130,9 @@ public class ThingsboardSecurityConfiguration { @Autowired private RateLimitProcessingFilter rateLimitProcessingFilter; + @Autowired + private AuthExceptionHandler authExceptionHandler; + @Bean protected PayloadSizeFilter payloadSizeFilter() { return new PayloadSizeFilter(maxPayloadSizeConfig); @@ -235,7 +239,8 @@ public class ThingsboardSecurityConfiguration { .addFilterBefore(buildJwtTokenAuthenticationProcessingFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterBefore(buildRefreshTokenProcessingFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterBefore(payloadSizeFilter(), UsernamePasswordAuthenticationFilter.class) - .addFilterAfter(rateLimitProcessingFilter, UsernamePasswordAuthenticationFilter.class); + .addFilterAfter(rateLimitProcessingFilter, UsernamePasswordAuthenticationFilter.class) + .addFilterBefore(authExceptionHandler, buildRestLoginProcessingFilter().getClass()); if (oauth2Configuration != null) { http.oauth2Login(login -> login .authorizationEndpoint(config -> config diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index d29d398dca..93bf5b35fe 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -437,14 +437,7 @@ public abstract class BaseController { } else if (exception instanceof AsyncRequestTimeoutException) { return new ThingsboardException("Request timeout", ThingsboardErrorCode.GENERAL); } else if (exception instanceof DataAccessException) { - if (!logControllerErrorStackTrace) { // not to log the error twice - log.warn("Database error: {} - {}", exception.getClass().getSimpleName(), ExceptionUtils.getRootCauseMessage(exception)); - } - if (cause instanceof ConstraintViolationException) { - return new ThingsboardException(ExceptionUtils.getRootCause(exception).getMessage(), ThingsboardErrorCode.BAD_REQUEST_PARAMS); - } else { - return new ThingsboardException("Database error", ThingsboardErrorCode.GENERAL); - } + return new ThingsboardException(exception, ThingsboardErrorCode.DATABASE); } else if (exception instanceof EntityVersionMismatchException) { return new ThingsboardException(exception.getMessage(), exception, ThingsboardErrorCode.VERSION_CONFLICT); } diff --git a/application/src/main/java/org/thingsboard/server/exception/ThingsboardErrorResponseHandler.java b/application/src/main/java/org/thingsboard/server/exception/ThingsboardErrorResponseHandler.java index 9a08441936..7be11e5748 100644 --- a/application/src/main/java/org/thingsboard/server/exception/ThingsboardErrorResponseHandler.java +++ b/application/src/main/java/org/thingsboard/server/exception/ThingsboardErrorResponseHandler.java @@ -21,7 +21,9 @@ import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.exception.ExceptionUtils; +import org.hibernate.exception.ConstraintViolationException; import org.springframework.boot.web.servlet.error.ErrorController; +import org.springframework.dao.DataAccessException; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatusCode; @@ -139,6 +141,8 @@ public class ThingsboardErrorResponseHandler extends ResponseEntityExceptionHand ThingsboardException thingsboardException = (ThingsboardException) exception; if (thingsboardException.getErrorCode() == ThingsboardErrorCode.SUBSCRIPTION_VIOLATION) { handleSubscriptionException((ThingsboardException) exception, response); + } else if (thingsboardException.getErrorCode() == ThingsboardErrorCode.DATABASE) { + handleDatabaseException(thingsboardException.getCause(), response); } else { handleThingsboardException((ThingsboardException) exception, response); } @@ -148,8 +152,10 @@ public class ThingsboardErrorResponseHandler extends ResponseEntityExceptionHand handleAccessDeniedException(response); } else if (exception instanceof AuthenticationException) { handleAuthenticationException((AuthenticationException) exception, response); - } else if (exception instanceof MaxPayloadSizeExceededException) { + } else if (exception instanceof MaxPayloadSizeExceededException) { handleMaxPayloadSizeExceededException(response, (MaxPayloadSizeExceededException) exception); + } else if (exception instanceof DataAccessException e) { + handleDatabaseException(e, response); } else { response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value()); JacksonUtil.writeValue(response.getWriter(), ThingsboardErrorResponse.of(exception.getMessage(), @@ -201,6 +207,17 @@ public class ThingsboardErrorResponseHandler extends ResponseEntityExceptionHand JacksonUtil.fromBytes(((HttpClientErrorException) subscriptionException.getCause()).getResponseBodyAsByteArray(), Object.class)); } + private void handleDatabaseException(Throwable databaseException, HttpServletResponse response) throws IOException { + ThingsboardErrorResponse errorResponse; + if (databaseException instanceof ConstraintViolationException) { + errorResponse = ThingsboardErrorResponse.of(ExceptionUtils.getRootCause(databaseException).getMessage(), ThingsboardErrorCode.BAD_REQUEST_PARAMS, HttpStatus.BAD_REQUEST); + } else { + log.warn("Database error: {} - {}", databaseException.getClass().getSimpleName(), ExceptionUtils.getRootCauseMessage(databaseException)); + errorResponse = ThingsboardErrorResponse.of("Database error", ThingsboardErrorCode.DATABASE, HttpStatus.INTERNAL_SERVER_ERROR); + } + writeResponse(errorResponse, response); + } + private void handleAccessDeniedException(HttpServletResponse response) throws IOException { response.setStatus(HttpStatus.FORBIDDEN.value()); JacksonUtil.writeValue(response.getWriter(), @@ -233,4 +250,11 @@ public class ThingsboardErrorResponseHandler extends ResponseEntityExceptionHand } } + // TODO: refactor this class to use this method instead of boilerplate JacksonUtil.writeValue(response.getWriter(), ... + private void writeResponse(ThingsboardErrorResponse errorResponse, HttpServletResponse response) throws IOException { + response.setContentType(MediaType.APPLICATION_JSON_VALUE); + response.setStatus(errorResponse.getStatus()); + JacksonUtil.writeValue(response.getWriter(), errorResponse); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/AuthExceptionHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/AuthExceptionHandler.java new file mode 100644 index 0000000000..27ed2996d1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/AuthExceptionHandler.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth; + +import jakarta.servlet.FilterChain; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.security.core.AuthenticationException; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; +import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; + +@Component +@RequiredArgsConstructor +@Slf4j +public class AuthExceptionHandler extends OncePerRequestFilter { + + private final ThingsboardErrorResponseHandler errorResponseHandler; + + @Override + protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) { + try { + filterChain.doFilter(request, response); + } catch (AuthenticationException e) { + throw e; + } catch (Exception e) { + errorResponseHandler.handle(e, response); + } + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/exception/ThingsboardErrorCode.java b/common/data/src/main/java/org/thingsboard/server/common/data/exception/ThingsboardErrorCode.java index 13c65fe6e6..deae86a9de 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/exception/ThingsboardErrorCode.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/exception/ThingsboardErrorCode.java @@ -31,7 +31,8 @@ public enum ThingsboardErrorCode { TOO_MANY_UPDATES(34), VERSION_CONFLICT(35), SUBSCRIPTION_VIOLATION(40), - PASSWORD_VIOLATION(45); + PASSWORD_VIOLATION(45), + DATABASE(46); private int errorCode; From ed5868d8e5d825d033d638a4da297ca1a397f285 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 9 May 2025 14:16:53 +0300 Subject: [PATCH 33/79] removed permission check for tenant --- .../server/controller/CalculatedFieldController.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 1c00988f6a..0dcf22e433 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -270,7 +270,10 @@ public class CalculatedFieldController extends BaseController { for (EntityId referencedEntityId : referencedEntityIds) { EntityType entityType = referencedEntityId.getEntityType(); switch (entityType) { - case TENANT, CUSTOMER, ASSET, DEVICE -> checkEntityId(referencedEntityId, Operation.READ); + case TENANT -> { + return; + } + case CUSTOMER, ASSET, DEVICE -> checkEntityId(referencedEntityId, Operation.READ); default -> throw new IllegalArgumentException("Calculated fields do not support '" + entityType + "' for referenced entities."); } From 9bb50ae355265dae626432855e87f4788fb3ddb8 Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Fri, 9 May 2025 14:55:28 +0300 Subject: [PATCH 34/79] UI: Refacoring hide clear event --- .../app/modules/home/components/event/event-table.component.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/event/event-table.component.ts b/ui-ngx/src/app/modules/home/components/event/event-table.component.ts index f51cf66d8a..53cb8e1b96 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table.component.ts @@ -156,7 +156,7 @@ export class EventTableComponent implements OnInit, AfterViewInit, OnDestroy { } ngAfterViewInit() { - this.isEmptyData$ = this.entitiesTable.dataSource.isEmpty().subscribe(value => this.eventTableConfig.hideClearEventAction = value); + this.isEmptyData$ = this.entitiesTable.dataSource.isEmpty().subscribe(value => this.eventTableConfig.hideClearEventAction = value || this.hideClearEventAction); } ngOnDestroy() { From b97c1888f147665f25b646b96b93a422841670d4 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 9 May 2025 19:05:28 +0300 Subject: [PATCH 35/79] EDQS readiness check; refactor API enabling --- .../controller/EntityQueryController.java | 7 +- .../AbstractCalculatedFieldStateService.java | 2 +- .../service/edqs/DefaultEdqsApiService.java | 26 ---- .../service/edqs/DefaultEdqsService.java | 137 +++++++++++++++--- .../src/main/resources/thingsboard.yml | 2 + .../EdqsEntityQueryControllerTest.java | 13 +- .../controller/EntityQueryControllerTest.java | 7 +- .../entitiy/EdqsEntityServiceTest.java | 9 +- .../server/common/data/edqs/EdqsState.java | 71 +++++++++ .../common/data/edqs/ToCoreEdqsMsg.java | 4 + .../common/data/edqs/ToCoreEdqsRequest.java | 6 - .../edqs/state/KafkaEdqsStateService.java | 21 ++- .../common/msg/edqs/EdqsApiService.java | 6 - .../server/common/msg/edqs/EdqsService.java | 5 + common/proto/src/main/proto/queue.proto | 1 + .../common/state/KafkaQueueStateService.java | 8 +- .../queue/common/state/QueueStateService.java | 21 ++- .../DefaultTbServiceInfoProvider.java | 11 ++ .../queue/discovery/DiscoveryService.java | 2 + .../discovery/DummyDiscoveryService.java | 9 ++ .../discovery/TbServiceInfoProvider.java | 2 + .../queue/discovery/ZkDiscoveryService.java | 12 ++ .../server/dao/entity/BaseEntityService.java | 10 +- .../dao/sql/query/DummyEdqsApiService.java | 15 -- .../dao/sql/query/DummyEdqsService.java | 11 ++ 25 files changed, 298 insertions(+), 120 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsState.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java index 7fd0b12077..3a93f318d1 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java @@ -29,6 +29,7 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; +import org.thingsboard.server.common.data.edqs.EdqsState; import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; @@ -149,9 +150,9 @@ public class EntityQueryController extends BaseController { } @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") - @GetMapping("/edqs/enabled") - public boolean isEdqsApiEnabled() { - return edqsApiService.isEnabled(); + @GetMapping("/edqs/state") + public EdqsState getEdqsState() { + return edqsService.getState(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java index 91c08ab6e0..f1cb25c6fa 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java @@ -72,7 +72,7 @@ public abstract class AbstractCalculatedFieldStateService implements CalculatedF @Override public void restore(QueueKey queueKey, Set partitions) { - stateService.update(queueKey, partitions); + stateService.update(queueKey, partitions, null); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java index c7e17b62ae..e0f0db82cc 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsApiService.java @@ -22,7 +22,6 @@ import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; @@ -51,11 +50,6 @@ public class DefaultEdqsApiService implements EdqsApiService { private final EdqsClientQueueFactory queueFactory; private TbQueueRequestTemplate, TbProtoQueueMsg> requestTemplate; - @Value("${queue.edqs.api.auto_enable:true}") - private boolean autoEnable; - - private Boolean apiEnabled = null; - @PostConstruct private void init() { requestTemplate = queueFactory.createEdqsRequestTemplate(); @@ -85,31 +79,11 @@ public class DefaultEdqsApiService implements EdqsApiService { }, MoreExecutors.directExecutor()); } - @Override - public boolean isEnabled() { - return Boolean.TRUE.equals(apiEnabled); - } - - @Override - public void setEnabled(boolean enabled) { - if (enabled) { - log.info("Enabling EDQS API"); - } else { - log.info("Disabling EDQS API"); - } - apiEnabled = enabled; - } - @Override public boolean isSupported() { return true; } - @Override - public boolean isAutoEnable() { - return autoEnable; - } - @PreDestroy private void stop() { requestTemplate.stop(); diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java index cd46c4ba96..024191a270 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java @@ -20,11 +20,13 @@ import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.AllArgsConstructor; import lombok.Data; +import lombok.Getter; import lombok.NoArgsConstructor; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; @@ -36,6 +38,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.EdqsEventType; import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.EdqsState; +import org.thingsboard.server.common.data.edqs.EdqsState.EdqsSyncStatus; import org.thingsboard.server.common.data.edqs.EdqsSyncRequest; import org.thingsboard.server.common.data.edqs.Entity; import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg; @@ -53,18 +57,22 @@ import org.thingsboard.server.edqs.processor.EdqsProducer; import org.thingsboard.server.edqs.state.EdqsPartitionService; import org.thingsboard.server.edqs.util.EdqsConverter; import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsCoreServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.discovery.DiscoveryService; import org.thingsboard.server.queue.discovery.HashPartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.environment.DistributedLock; import org.thingsboard.server.queue.environment.DistributedLockService; import org.thingsboard.server.queue.provider.EdqsClientQueueFactory; import org.thingsboard.server.queue.util.AfterStartUp; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @Service @@ -80,25 +88,36 @@ public class DefaultEdqsService implements EdqsService { private final DistributedLockService distributedLockService; private final AttributesService attributesService; private final EdqsPartitionService edqsPartitionService; - private final TopicService topicService; private final TbServiceInfoProvider serviceInfoProvider; + private final DiscoveryService discoveryService; @Autowired @Lazy private TbClusterService clusterService; @Autowired @Lazy private HashPartitionService hashPartitionService; + @Value("${queue.edqs.api.auto_enable:true}") + private boolean autoEnableApi; + @Value("${queue.edqs.readiness_check_interval:60000}") + private int edqsReadinessCheckInterval; + private EdqsProducer eventsProducer; private ExecutorService executor; + private ScheduledExecutorService scheduler; private DistributedLock syncLock; + @Getter + private EdqsState state; + @PostConstruct private void init() { executor = ThingsBoardExecutors.newWorkStealingPool(12, getClass()); + scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-check"); eventsProducer = EdqsProducer.builder() .producer(queueFactory.createEdqsEventsProducer()) .partitionService(edqsPartitionService) .build(); syncLock = distributedLockService.getLock("edqs_sync"); + state = new EdqsState(); } @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) @@ -106,6 +125,26 @@ public class DefaultEdqsService implements EdqsService { if (!serviceInfoProvider.isService(ServiceType.TB_CORE)) { return; } + if (edqsApiService.isSupported()) { + scheduler.scheduleWithFixedDelay(() -> { + if (!hashPartitionService.isSystemPartitionMine(ServiceType.TB_CORE)) { + return; + } + + List servers = new ArrayList<>(discoveryService.getOtherServers()); + servers.add(serviceInfoProvider.getServiceInfo()); + + List readyEdqsServers = servers.stream() + .filter(serviceInfo -> serviceInfo.getServiceTypesList().contains(ServiceType.EDQS.name())) + .filter(ServiceInfo::getReady) + .toList(); + boolean changed = state.setEdqsReady(!readyEdqsServers.isEmpty()); + if (changed) { + broadcastEdqsReady(state.getEdqsReady()); + } + }, 0, edqsReadinessCheckInterval, TimeUnit.MILLISECONDS); + } + executor.submit(() -> { try { EdqsSyncState syncState = getSyncState(); @@ -115,9 +154,9 @@ public class DefaultEdqsService implements EdqsService { .syncRequest(new EdqsSyncRequest()) .build()); } - } else if (edqsApiService.isSupported() && edqsApiService.isAutoEnable()) { + } else { // only if topic/RocksDB is not empty and sync is finished - edqsApiService.setEnabled(true); + onSyncStatusUpdate(EdqsSyncStatus.FINISHED); } } catch (Throwable e) { log.error("Failed to start EDQS service", e); @@ -131,7 +170,10 @@ public class DefaultEdqsService implements EdqsService { if (request.getSyncRequest() != null) { saveSyncState(EdqsSyncStatus.REQUESTED); } - broadcast(request.toInternalMsg()); + broadcast(ToCoreEdqsMsg.builder() + .syncRequest(request.getSyncRequest()) + .apiEnabled(request.getApiEnabled()) + .build()); } @Override @@ -140,7 +182,13 @@ public class DefaultEdqsService implements EdqsService { log.info("Processing system msg {}", msg); try { if (msg.getApiEnabled() != null) { - edqsApiService.setEnabled(msg.getApiEnabled()); + state.setApiEnabled(msg.getApiEnabled()); + } + if (msg.getEdqsReady() != null) { + onEdqsReady(msg.getEdqsReady()); + } + if (msg.getSyncStatus() != null) { + onSyncStatusUpdate(msg.getSyncStatus()); } if (msg.getSyncRequest() != null) { @@ -154,23 +202,16 @@ public class DefaultEdqsService implements EdqsService { return; } } - saveSyncState(EdqsSyncStatus.STARTED); + edqsSyncService.sync(); - saveSyncState(EdqsSyncStatus.FINISHED); - if (edqsApiService.isSupported()) - if (edqsApiService.isAutoEnable()) { - log.info("EDQS sync is finished, auto-enabling API"); - broadcast(ToCoreEdqsMsg.builder() - .apiEnabled(Boolean.TRUE) - .build()); - } else { - log.info("EDQS sync is finished, but leaving API disabled"); - } + saveSyncState(EdqsSyncStatus.FINISHED); + broadcastSyncStatusUpdate(EdqsSyncStatus.FINISHED); } catch (Exception e) { log.error("Failed to complete sync", e); saveSyncState(EdqsSyncStatus.FAILED); + broadcastSyncStatusUpdate(EdqsSyncStatus.FAILED); } finally { syncLock.unlock(); } @@ -181,6 +222,60 @@ public class DefaultEdqsService implements EdqsService { }); } + private void broadcastEdqsReady(boolean ready) { + broadcast(ToCoreEdqsMsg.builder() + .edqsReady(ready) + .build()); + } + + private void onEdqsReady(boolean ready) { + state.setEdqsReady(ready); + checkState(); + } + + private void broadcastSyncStatusUpdate(EdqsSyncStatus status) { + broadcast(ToCoreEdqsMsg.builder() + .syncStatus(status) + .build()); + } + + private void onSyncStatusUpdate(EdqsSyncStatus status) { + state.setSyncStatus(status); + checkState(); + } + + private void checkState() { + if (!edqsApiService.isSupported()) { + log.info("New state: {}. EDQS API not supported", state); + return; + } + + if (state.isApiReady()) { + if (autoEnableApi) { + if (state.getApiEnabled() == null) { + state.setApiEnabled(true); + log.info("New state: {}. Auto-enabled EDQS API", state); + } else { + log.info("New state: {}. API mode left as is", state); + } + } else { + log.info("New state: {}. API auto-enabling is disabled", state); + } + } else { + if (state.isApiEnabled()) { + state.setApiEnabled(false); + log.info("New state: {}. Disabled EDQS API", state); + } else { + log.info("New state: {}. API left disabled", state); + } + } + } + + @Override + public boolean isApiEnabled() { + return state.isApiEnabled(); + } + @Override public void onUpdate(TenantId tenantId, EntityId entityId, Object entity) { EntityType entityType = entityId.getEntityType(); @@ -278,6 +373,7 @@ public class DefaultEdqsService implements EdqsService { @PreDestroy private void stop() { executor.shutdown(); + scheduler.shutdownNow(); eventsProducer.stop(); } @@ -288,11 +384,4 @@ public class DefaultEdqsService implements EdqsService { private EdqsSyncStatus status; } - private enum EdqsSyncStatus { - REQUESTED, - STARTED, - FINISHED, - FAILED - } - } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 6ad3386c4f..bebddd3f87 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1755,6 +1755,8 @@ queue: supported: "${TB_EDQS_API_SUPPORTED:false}" # Whether to auto-enable EDQS API (if queue.edqs.api.supported is true) when sync of data to Kafka is finished auto_enable: "${TB_EDQS_API_AUTO_ENABLE:true}" + # Interval in milliseconds to check for ready EDQS servers + readiness_check_interval: "${TB_EDQS_READINESS_CHECK_INTERVAL_MS:60000}" # Mode of EDQS: local (for monolith) or remote (with separate EDQS microservices) mode: "${TB_EDQS_MODE:local}" local: diff --git a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java index 153ec2d26f..b84a324bf7 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java @@ -23,9 +23,8 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; -import org.thingsboard.server.common.msg.edqs.EdqsApiService; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.edqs.state.EdqsStateService; import org.thingsboard.server.edqs.util.EdqsRocksDb; import java.util.concurrent.TimeUnit; @@ -39,22 +38,20 @@ import static org.awaitility.Awaitility.await; "queue.edqs.sync.enabled=true", "queue.edqs.api.supported=true", "queue.edqs.api.auto_enable=true", - "queue.edqs.mode=local" + "queue.edqs.mode=local", + "queue.edqs.readiness_check_interval=1000" }) public class EdqsEntityQueryControllerTest extends EntityQueryControllerTest { @Autowired - private EdqsApiService edqsApiService; - - @Autowired - private EdqsStateService edqsStateService; + private EdqsService edqsService; @MockBean // so that we don't do backup for tests private EdqsRocksDb edqsRocksDb; @Before public void before() { - await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsApiService.isEnabled() && edqsStateService.isReady()); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsService.getState().isApiEnabled()); } @Override diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java index 29a208a805..26c02e8704 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java @@ -22,6 +22,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.TestPropertySource; import org.springframework.test.web.servlet.ResultActions; import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; import org.thingsboard.common.util.JacksonUtil; @@ -78,6 +79,10 @@ import static org.awaitility.Awaitility.await; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @DaoSqlTest +@TestPropertySource(properties = { + "queue.edqs.sync.enabled=true", // only enabling sync + "queue.edqs.api.supported=false", +}) public class EntityQueryControllerTest extends AbstractControllerTest { private static final String CUSTOMER_USER_EMAIL = "entityQueryCustomer@thingsboard.org"; @@ -803,7 +808,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { //assign dashboard doPost("/api/customer/" + savedCustomer.getId().getId().toString() - + "/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class); + + "/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class); // check entity data query by customer User customerUser = new User(); diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java index 5264e69bd3..2f244807bd 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java @@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.RelationsQueryFilter; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; -import org.thingsboard.server.common.msg.edqs.EdqsApiService; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.edqs.util.EdqsRocksDb; @@ -53,19 +53,20 @@ import static org.awaitility.Awaitility.await; "queue.edqs.sync.enabled=true", "queue.edqs.api.supported=true", "queue.edqs.api.auto_enable=true", - "queue.edqs.mode=local" + "queue.edqs.mode=local", + "queue.edqs.readiness_check_interval=1000" }) public class EdqsEntityServiceTest extends EntityServiceTest { @Autowired - private EdqsApiService edqsApiService; + private EdqsService edqsService; @MockBean private EdqsRocksDb edqsRocksDb; @Before public void beforeEach() { - await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsApiService.isEnabled()); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsService.isApiEnabled()); } // sql implementation has a bug with data duplication, edqs implementation returns correct value diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsState.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsState.java new file mode 100644 index 0000000000..ccd5406f93 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsState.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.BooleanUtils; + +@Getter +@NoArgsConstructor +public class EdqsState { + + private Boolean edqsReady; + private EdqsSyncStatus syncStatus; + + private Boolean apiEnabled; // null until auto-enabled or set manually + + public boolean setEdqsReady(boolean ready) { + boolean changed = BooleanUtils.toBooleanDefaultIfNull(this.edqsReady, false) != ready; + this.edqsReady = ready; + return changed; + } + + public void setSyncStatus(EdqsSyncStatus syncStatus) { + this.syncStatus = syncStatus; + } + + public boolean setApiEnabled(boolean apiEnabled) { + boolean changed = BooleanUtils.toBooleanDefaultIfNull(this.apiEnabled, false) != apiEnabled; + this.apiEnabled = apiEnabled; + return changed; + } + + public boolean isApiReady() { + return edqsReady && syncStatus == EdqsSyncStatus.FINISHED; + } + + public boolean isApiEnabled() { + return apiEnabled != null && apiEnabled; + } + + @Override + public String toString() { + return '[' + + "EDQS ready: " + edqsReady + + ", sync status: " + syncStatus + + ", API enabled: " + apiEnabled + + ']'; + } + + public enum EdqsSyncStatus { + REQUESTED, + STARTED, + FINISHED, + FAILED + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsMsg.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsMsg.java index 78bebba20a..d581a5d17c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsMsg.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsMsg.java @@ -19,6 +19,7 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.edqs.EdqsState.EdqsSyncStatus; @Data @AllArgsConstructor @@ -29,4 +30,7 @@ public class ToCoreEdqsMsg { private EdqsSyncRequest syncRequest; private Boolean apiEnabled; + private EdqsSyncStatus syncStatus; + private Boolean edqsReady; + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsRequest.java index c4f262fbf0..44bbfa2cc9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsRequest.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.common.data.edqs; -import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -30,9 +29,4 @@ public class ToCoreEdqsRequest { private EdqsSyncRequest syncRequest; private Boolean apiEnabled; - @JsonIgnore - public ToCoreEdqsMsg toInternalMsg() { - return new ToCoreEdqsMsg(syncRequest, apiEnabled); - } - } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java index e8ede39d2c..7ad0258cbb 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java @@ -34,7 +34,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; import org.thingsboard.server.queue.common.state.KafkaQueueStateService; -import org.thingsboard.server.queue.common.state.QueueStateService; +import org.thingsboard.server.queue.discovery.DiscoveryService; import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.edqs.EdqsConfig; import org.thingsboard.server.queue.edqs.EdqsExecutors; @@ -61,20 +61,22 @@ public class KafkaEdqsStateService implements EdqsStateService { private final EdqsConfig config; private final EdqsPartitionService partitionService; private final KafkaEdqsQueueFactory queueFactory; + private final DiscoveryService discoveryService; private final EdqsExecutors edqsExecutors; @Autowired @Lazy private EdqsProcessor edqsProcessor; private PartitionedQueueConsumerManager> stateConsumer; - private QueueStateService, TbProtoQueueMsg> queueStateService; + private KafkaQueueStateService, TbProtoQueueMsg> queueStateService; private QueueConsumerManager> eventsToBackupConsumer; private EdqsProducer stateProducer; private final VersionsStore versionsStore = new VersionsStore(); private final AtomicInteger stateReadCount = new AtomicInteger(); private final AtomicInteger eventsReadCount = new AtomicInteger(); - private Boolean ready; + + private boolean ready = false; @Override public void init(PartitionedQueueConsumerManager> eventConsumer, List> otherConsumers) { @@ -187,7 +189,10 @@ public class KafkaEdqsStateService implements EdqsStateService { eventsToBackupConsumer.subscribe(allPartitions); eventsToBackupConsumer.launch(); } - queueStateService.update(new QueueKey(ServiceType.EDQS), partitions); + queueStateService.update(new QueueKey(ServiceType.EDQS), partitions, () -> { + ready = true; + discoveryService.setReady(true); + }); } @Override @@ -197,13 +202,7 @@ public class KafkaEdqsStateService implements EdqsStateService { @Override public boolean isReady() { - if (ready == null) { - Set partitionsInProgress = queueStateService.getPartitionsInProgress(); - if (partitionsInProgress != null && partitionsInProgress.isEmpty()) { - ready = true; // once true - always true, not to change readiness status on each repartitioning - } - } - return ready != null && ready; + return ready; } private TenantId getTenantId(ToEdqsMsg edqsMsg) { diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsApiService.java b/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsApiService.java index 05864fe863..0fe9971712 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsApiService.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsApiService.java @@ -25,12 +25,6 @@ public interface EdqsApiService { ListenableFuture processRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request); - boolean isEnabled(); - - void setEnabled(boolean enabled); - boolean isSupported(); - boolean isAutoEnable(); - } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsService.java b/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsService.java index 32ff57e3e0..faad9a0f4a 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsService.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.common.msg.edqs; import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.EdqsState; import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg; import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; import org.thingsboard.server.common.data.id.EntityId; @@ -36,4 +37,8 @@ public interface EdqsService { void processSystemMsg(ToCoreEdqsMsg request); + boolean isApiEnabled(); + + EdqsState getState(); + } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 938a1692ae..e874c81444 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -87,6 +87,7 @@ message ServiceInfo { SystemInfoProto systemInfo = 10; repeated string assignedTenantProfiles = 11; string label = 12; + bool ready = 13; } message SystemInfoProto { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/state/KafkaQueueStateService.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/state/KafkaQueueStateService.java index d8d0c8e0d2..2a38c9a86c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/state/KafkaQueueStateService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/state/KafkaQueueStateService.java @@ -26,6 +26,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Supplier; import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTopic; @@ -36,6 +37,8 @@ public class KafkaQueueStateService private final PartitionedQueueConsumerManager stateConsumer; private final Supplier> eventsStartOffsetsProvider; + private final Set partitionsInProgress = ConcurrentHashMap.newKeySet(); + @Builder public KafkaQueueStateService(PartitionedQueueConsumerManager eventConsumer, PartitionedQueueConsumerManager stateConsumer, @@ -47,7 +50,7 @@ public class KafkaQueueStateService } @Override - protected void addPartitions(QueueKey queueKey, Set partitions) { + protected void addPartitions(QueueKey queueKey, Set partitions, Runnable whenAllProcessed) { Map eventsStartOffsets = eventsStartOffsetsProvider != null ? eventsStartOffsetsProvider.get() : null; // remembering the offsets before subscribing to states Set statePartitions = withTopic(partitions, stateConsumer.getTopic()); @@ -60,6 +63,9 @@ public class KafkaQueueStateService log.info("Finished partition {} (still in progress: {})", statePartition, partitionsInProgress); if (partitionsInProgress.isEmpty()) { log.info("All partitions processed"); + if (whenAllProcessed != null) { + whenAllProcessed.run(); + } } TopicPartitionInfo eventPartition = statePartition.withTopic(eventConsumer.getTopic()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/state/QueueStateService.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/state/QueueStateService.java index 61b81707ce..e58d5eb036 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/state/QueueStateService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/state/QueueStateService.java @@ -28,7 +28,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -42,8 +41,6 @@ public abstract class QueueStateService> partitions = new HashMap<>(); - protected final Set partitionsInProgress = ConcurrentHashMap.newKeySet(); - protected boolean initialized; protected final ReadWriteLock partitionsLock = new ReentrantReadWriteLock(); @@ -52,7 +49,7 @@ public abstract class QueueStateService newPartitions) { + public void update(QueueKey queueKey, Set newPartitions, Runnable whenAllProcessed) { newPartitions = withTopic(newPartitions, eventConsumer.getTopic()); var writeLock = partitionsLock.writeLock(); writeLock.lock(); @@ -74,12 +71,18 @@ public abstract class QueueStateService partitions) { + protected void addPartitions(QueueKey queueKey, Set partitions, Runnable whenAllProcessed) { + if (whenAllProcessed != null) { + whenAllProcessed.run(); + } eventConsumer.addPartitions(partitions); for (PartitionedQueueConsumerManager consumer : otherConsumers) { consumer.addPartitions(withTopic(partitions, consumer.getTopic())); @@ -114,10 +117,6 @@ public abstract class QueueStateService getPartitionsInProgress() { - return initialized ? partitionsInProgress : null; - } - public void stop() { eventConsumer.stop(); eventConsumer.awaitStop(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index 609d3f8eee..c2705cb1ab 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -68,6 +68,8 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { private List serviceTypes; private ServiceInfo serviceInfo; + private boolean ready = true; + @PostConstruct public void init() { if (StringUtils.isEmpty(serviceId)) { @@ -87,6 +89,7 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { assignedTenantProfiles = Collections.emptySet(); } if (serviceTypes.contains(ServiceType.EDQS)) { + ready = false; if (StringUtils.isBlank(edqsConfig.getLabel())) { edqsConfig.setLabel(serviceId); } @@ -128,9 +131,17 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { builder.addAllAssignedTenantProfiles(assignedTenantProfiles.stream().map(UUID::toString).collect(Collectors.toList())); } builder.setLabel(edqsConfig.getLabel()); + builder.setReady(ready); return serviceInfo = builder.build(); } + @Override + public boolean setReady(boolean ready) { + boolean changed = this.ready != ready; + this.ready = ready; + return changed; + } + private TransportProtos.SystemInfoProto getCurrentSystemInfoProto() { TransportProtos.SystemInfoProto.Builder builder = TransportProtos.SystemInfoProto.newBuilder(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java index 5d309014dd..28d61bde0f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java @@ -25,4 +25,6 @@ public interface DiscoveryService { boolean isMonolith(); + void setReady(boolean ready); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java index 442b845e81..8a4e9102e2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java @@ -55,4 +55,13 @@ public class DummyDiscoveryService implements DiscoveryService { public boolean isMonolith() { return true; } + + @Override + public void setReady(boolean ready) { + boolean changed = serviceInfoProvider.setReady(ready); + if (changed) { + serviceInfoProvider.generateNewServiceInfoWithCurrentSystemInfo(); + } + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java index 51a6d808dd..d7182ca2eb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java @@ -35,4 +35,6 @@ public interface TbServiceInfoProvider { Set getAssignedTenantProfiles(); + boolean setReady(boolean ready); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index cf9f27ee39..2d9ac3630e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -177,6 +177,18 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi } } + @Override + public void setReady(boolean ready) { + boolean changed = serviceInfoProvider.setReady(ready); + if (changed) { + try { + publishCurrentServer(); + } catch (Exception e) { + log.error("Failed to update server readiness status", e); + } + } + } + private boolean currentServerExists() { if (nodePath == null) { return false; diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index ee9320cce3..a000119235 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -46,6 +46,7 @@ import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.RelationsQueryFilter; import org.thingsboard.server.common.msg.edqs.EdqsApiService; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.common.stats.EdqsStatsService; import org.thingsboard.server.dao.exception.IncorrectParameterException; @@ -86,6 +87,9 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe @Lazy EntityServiceRegistry entityServiceRegistry; + @Autowired + private EdqsService edqsService; + @Autowired @Lazy private EdqsApiService edqsApiService; @@ -102,7 +106,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe long startNs = System.nanoTime(); Long result; - if (edqsApiService.isEnabled() && validForEdqs(query) && !tenantId.isSysTenantId()) { + if (edqsService.isApiEnabled() && validForEdqs(query) && !tenantId.isSysTenantId()) { EdqsRequest request = EdqsRequest.builder() .entityCountQuery(query) .build(); @@ -124,7 +128,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe long startNs = System.nanoTime(); PageData result; - if (edqsApiService.isEnabled() && validForEdqs(query)) { + if (edqsService.isApiEnabled() && validForEdqs(query)) { EdqsRequest request = EdqsRequest.builder() .entityDataQuery(query) .build(); @@ -285,7 +289,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe } if ((query.getEntityFields() == null || query.getEntityFields().isEmpty()) && - (query.getLatestValues() == null || query.getLatestValues().isEmpty())) { + (query.getLatestValues() == null || query.getLatestValues().isEmpty())) { return false; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsApiService.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsApiService.java index e486d3b645..f50bb9a68c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsApiService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsApiService.java @@ -35,24 +35,9 @@ public class DummyEdqsApiService implements EdqsApiService { throw new UnsupportedOperationException(); } - @Override - public boolean isEnabled() { - return false; - } - - @Override - public void setEnabled(boolean enabled) { - log.warn("Got request to enable EDQS API, but it isn't supported", new RuntimeException("stacktrace")); - } - @Override public boolean isSupported() { return false; } - @Override - public boolean isAutoEnable() { - return false; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java index 514e07c323..707273a11b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java @@ -19,6 +19,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.EdqsState; import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg; import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; import org.thingsboard.server.common.data.id.EntityId; @@ -47,4 +48,14 @@ public class DummyEdqsService implements EdqsService { @Override public void processSystemMsg(ToCoreEdqsMsg request) {} + @Override + public boolean isApiEnabled() { + return getState().isApiEnabled(); + } + + @Override + public EdqsState getState() { + return new EdqsState(); + } + } From 8b00da12c4ae5434d0b6c0033caf3b6ffb53e6b8 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Sat, 10 May 2025 12:16:25 +0200 Subject: [PATCH 36/79] tools cassandra-all bumped to v5 --- pom.xml | 7 +------ .../thingsboard/client/tools/migrator/WriterBuilder.java | 6 +++--- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index f95e9df9b1..e5d6c0e165 100755 --- a/pom.xml +++ b/pom.xml @@ -57,7 +57,7 @@ 0.10 4.17.0 4.2.25 - 3.11.17 + 5.0.4 33.1.0-jre 3.1.8 3.14.0 @@ -1827,11 +1827,6 @@ cassandra-all ${cassandra-all.version} - - org.apache.cassandra - cassandra-thrift - ${cassandra-all.version} - org.junit.vintage junit-vintage-engine diff --git a/tools/src/main/java/org/thingsboard/client/tools/migrator/WriterBuilder.java b/tools/src/main/java/org/thingsboard/client/tools/migrator/WriterBuilder.java index b7f8524031..80fdad3c1c 100644 --- a/tools/src/main/java/org/thingsboard/client/tools/migrator/WriterBuilder.java +++ b/tools/src/main/java/org/thingsboard/client/tools/migrator/WriterBuilder.java @@ -59,7 +59,7 @@ public class WriterBuilder { public static CQLSSTableWriter getTsWriter(File dir) { return CQLSSTableWriter.builder() - .inDirectory(dir) + .inDirectory(dir.getAbsolutePath()) .forTable(tsSchema) .using("INSERT INTO thingsboard.ts_kv_cf (entity_type, entity_id, key, partition, ts, bool_v, str_v, long_v, dbl_v, json_v) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") @@ -68,7 +68,7 @@ public class WriterBuilder { public static CQLSSTableWriter getLatestWriter(File dir) { return CQLSSTableWriter.builder() - .inDirectory(dir) + .inDirectory(dir.getAbsolutePath()) .forTable(latestSchema) .using("INSERT INTO thingsboard.ts_kv_latest_cf (entity_type, entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)") @@ -77,7 +77,7 @@ public class WriterBuilder { public static CQLSSTableWriter getPartitionWriter(File dir) { return CQLSSTableWriter.builder() - .inDirectory(dir) + .inDirectory(dir.getAbsolutePath()) .forTable(partitionSchema) .using("INSERT INTO thingsboard.ts_kv_partitions_cf (entity_type, entity_id, key, partition) " + "VALUES (?, ?, ?, ?)") From 5111a07a1154e92263e11138bab8078acd54a8c9 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Sat, 10 May 2025 12:23:05 +0200 Subject: [PATCH 37/79] addressing vulnerabilities on high and critical --- pom.xml | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index e5d6c0e165..cd2078e70a 100755 --- a/pom.xml +++ b/pom.xml @@ -42,13 +42,14 @@ 4.0.2 2.4.0-b180830.0359 4.0.5 - 10.1.39 + 10.1.40 + 2.5.2 3.2.12 3.2.12 3.2.12 6.1.15 6.2.11 - 6.2.8 + 6.3.8 5.1.5 0.12.5 2.0.13 @@ -102,7 +103,7 @@ 1.19.0 1.78.1 2.0.1 - 42.7.3 + 42.7.5 org/thingsboard/server/gen/**/*, org/thingsboard/server/extensions/core/plugin/telemetry/gen/**/* @@ -112,7 +113,7 @@ - 3.7.1 + 3.7.2 8.10.1 3.5.3 2.2 @@ -1163,6 +1164,13 @@ tomcat-embed-websocket ${tomcat.version} + + + net.minidev + json-smart + ${net.minidev.json-smart} + + org.springframework.boot spring-boot-starter @@ -1183,6 +1191,18 @@ spring-security-oauth2-jose ${spring-security.version} + + + org.springframework.security + spring-security-config + ${spring-security.version} + + + org.springframework.security + spring-security-web + ${spring-security.version} + + org.springframework spring-core From 9d72f5f4346d621ab4d8752b331e0990cb14eee2 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Sat, 10 May 2025 12:24:24 +0200 Subject: [PATCH 38/79] bump californium version from 2.6.1 to 2.7.4 to fix vulnerability and not break the monitoring module --- monitoring/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/pom.xml b/monitoring/pom.xml index 4f59cbc975..4ef6c775bf 100644 --- a/monitoring/pom.xml +++ b/monitoring/pom.xml @@ -42,7 +42,7 @@ ThingsBoard Monitoring Service org.thingsboard.monitoring.ThingsboardMonitoringApplication - 2.6.1 + 2.7.4 2.0.0-M4 From dec61b8ac28be647563f33ad2565bde0feef5039 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Sat, 10 May 2025 12:25:53 +0200 Subject: [PATCH 39/79] cassandra thrift removed after cassandra-all bump for module tools --- tools/pom.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/pom.xml b/tools/pom.xml index 0286c135e4..7e2c44ddd6 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -55,10 +55,6 @@ org.apache.cassandra cassandra-all - - org.apache.cassandra - cassandra-thrift - commons-io commons-io From a3c7084d7fc98492386be0217a5d7ff4c3204493 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Sat, 10 May 2025 14:24:04 +0200 Subject: [PATCH 40/79] refactored obsolete com.github.java-json-tools:json-schema-validator with actively updated com.networknt:json-schema-validator --- common/cluster-api/pom.xml | 4 - common/dao-api/pom.xml | 4 - dao/pom.xml | 4 + .../BaseComponentDescriptorService.java | 20 ++-- .../BaseComponentDescriptorServiceTest.java | 98 +++++++++++++++++++ pom.xml | 10 +- 6 files changed, 116 insertions(+), 24 deletions(-) create mode 100644 dao/src/test/java/org/thingsboard/server/dao/component/BaseComponentDescriptorServiceTest.java diff --git a/common/cluster-api/pom.xml b/common/cluster-api/pom.xml index ee9c1eae75..75048197d3 100644 --- a/common/cluster-api/pom.xml +++ b/common/cluster-api/pom.xml @@ -60,10 +60,6 @@ jakarta.annotation jakarta.annotation-api - - com.github.java-json-tools - json-schema-validator - org.slf4j slf4j-api diff --git a/common/dao-api/pom.xml b/common/dao-api/pom.xml index d7142e723d..d3027356c3 100644 --- a/common/dao-api/pom.xml +++ b/common/dao-api/pom.xml @@ -56,10 +56,6 @@ jakarta.annotation jakarta.annotation-api - - com.github.java-json-tools - json-schema-validator - org.slf4j slf4j-api diff --git a/dao/pom.xml b/dao/pom.xml index f203343c89..9618c38386 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -59,6 +59,10 @@ org.thingsboard.common util + + com.networknt + json-schema-validator + org.slf4j slf4j-api diff --git a/dao/src/main/java/org/thingsboard/server/dao/component/BaseComponentDescriptorService.java b/dao/src/main/java/org/thingsboard/server/dao/component/BaseComponentDescriptorService.java index 5acf1b0f97..2c5d6fb3c5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/component/BaseComponentDescriptorService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/component/BaseComponentDescriptorService.java @@ -16,10 +16,10 @@ package org.thingsboard.server.dao.component; import com.fasterxml.jackson.databind.JsonNode; -import com.github.fge.jsonschema.core.exceptions.ProcessingException; -import com.github.fge.jsonschema.core.report.ProcessingReport; -import com.github.fge.jsonschema.main.JsonSchemaFactory; -import com.github.fge.jsonschema.main.JsonValidator; +import com.networknt.schema.JsonSchema; +import com.networknt.schema.JsonSchemaFactory; +import com.networknt.schema.SpecVersion; +import com.networknt.schema.ValidationMessage; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -36,6 +36,7 @@ import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.Validator; import java.util.Optional; +import java.util.Set; /** * @author Andrew Shvayka @@ -89,15 +90,18 @@ public class BaseComponentDescriptorService implements ComponentDescriptorServic @Override public boolean validate(TenantId tenantId, ComponentDescriptor component, JsonNode configuration) { - JsonValidator validator = JsonSchemaFactory.byDefault().getValidator(); try { if (!component.getConfigurationDescriptor().has("schema")) { throw new DataValidationException("Configuration descriptor doesn't contain schema property!"); } JsonNode configurationSchema = component.getConfigurationDescriptor().get("schema"); - ProcessingReport report = validator.validate(configurationSchema, configuration); - return report.isSuccess(); - } catch (ProcessingException e) { + + JsonSchemaFactory factory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V4); + JsonSchema schema = factory.getSchema(configurationSchema); + + Set validationMessages = schema.validate(configuration); + return validationMessages.isEmpty(); + } catch (Exception e) { throw new IncorrectParameterException(e.getMessage(), e); } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/component/BaseComponentDescriptorServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/component/BaseComponentDescriptorServiceTest.java new file mode 100644 index 0000000000..d76c11f708 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/component/BaseComponentDescriptorServiceTest.java @@ -0,0 +1,98 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.component; + +import com.fasterxml.jackson.databind.JsonNode; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentClusteringMode; +import org.thingsboard.server.common.data.plugin.ComponentDescriptor; +import org.thingsboard.server.common.data.plugin.ComponentScope; +import org.thingsboard.server.common.data.plugin.ComponentType; +import org.thingsboard.server.dao.exception.IncorrectParameterException; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class BaseComponentDescriptorServiceTest { + + private BaseComponentDescriptorService service; + private ComponentDescriptor componentDescriptor; + private TenantId tenantId; + + @BeforeEach + void setUp() { + service = Mockito.spy(BaseComponentDescriptorService.class); + tenantId = TenantId.SYS_TENANT_ID; + + // Create a simple component descriptor + componentDescriptor = new ComponentDescriptor(); + componentDescriptor.setType(ComponentType.ACTION); + componentDescriptor.setScope(ComponentScope.TENANT); + componentDescriptor.setClusteringMode(ComponentClusteringMode.ENABLED); + componentDescriptor.setName("Test Component"); + componentDescriptor.setClazz("org.thingsboard.test.TestComponent"); + + // Create configuration descriptor with schema from JSON string + String configDescriptorJson = """ + { + "schema": { + "type": "object", + "properties": { + "testField": { + "type": "string" + } + }, + "required": ["testField"] + } + }"""; + + componentDescriptor.setConfigurationDescriptor(JacksonUtil.toJsonNode(configDescriptorJson)); + } + + @Test + void testValidate() { + // Create valid configuration from JSON string + String validConfigJson = "{\"testField\": \"test value\"}"; + JsonNode validConfig = JacksonUtil.toJsonNode(validConfigJson); + + // Create invalid configuration (missing required field) from JSON string + String invalidConfigJson = "{}"; + JsonNode invalidConfig = JacksonUtil.toJsonNode(invalidConfigJson); + + // Test valid configuration + boolean validResult = service.validate(tenantId, componentDescriptor, validConfig); + assertTrue(validResult, "Valid configuration should pass validation"); + + // Test invalid configuration + boolean invalidResult = service.validate(tenantId, componentDescriptor, invalidConfig); + assertFalse(invalidResult, "Invalid configuration should fail validation"); + + // Test with component descriptor without schema + ComponentDescriptor noSchemaDescriptor = new ComponentDescriptor(componentDescriptor); + noSchemaDescriptor.setConfigurationDescriptor(JacksonUtil.toJsonNode("{}")); + + // Should throw exception when schema is missing + assertThrows(IncorrectParameterException.class, () -> { + service.validate(tenantId, noSchemaDescriptor, validConfig); + }, "Should throw exception when schema is missing"); + } + +} diff --git a/pom.xml b/pom.xml index cd2078e70a..0260dc09be 100755 --- a/pom.xml +++ b/pom.xml @@ -75,7 +75,7 @@ 2.17.2 1.7.0 4.4.0 - 2.2.14 + 1.5.6 0.6.12 3.12.1 2.0.0-M15 @@ -1620,15 +1620,9 @@ ${auth0-jwt.version} - com.github.java-json-tools + com.networknt json-schema-validator ${json-schema-validator.version} - - - com.sun.mail - mailapi - - org.eclipse.leshan From d608b87f945337908b4c731a10116eab35b080b0 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Sat, 10 May 2025 15:25:59 +0200 Subject: [PATCH 41/79] test scope mockserver with no dependency to not affect transitive dependencies --- pom.xml | 4 ++-- rule-engine/rule-engine-components/pom.xml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 0260dc09be..608a46584a 100755 --- a/pom.xml +++ b/pom.xml @@ -2233,7 +2233,7 @@ org.mock-server - mockserver-netty + mockserver-netty-no-dependencies ${mock-server.version} test @@ -2245,7 +2245,7 @@ org.mock-server - mockserver-client-java + mockserver-client-java-no-dependencies ${mock-server.version} test diff --git a/rule-engine/rule-engine-components/pom.xml b/rule-engine/rule-engine-components/pom.xml index 008c422dbf..b975f7c624 100644 --- a/rule-engine/rule-engine-components/pom.xml +++ b/rule-engine/rule-engine-components/pom.xml @@ -141,12 +141,12 @@ org.mock-server - mockserver-netty + mockserver-netty-no-dependencies test org.mock-server - mockserver-client-java + mockserver-client-java-no-dependencies test From 99d2d1e03311cbba6eadf6e2959d122ec4258e67 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Sat, 10 May 2025 15:30:08 +0200 Subject: [PATCH 42/79] Monitoring COAP Leshan Dependency Upgrade --- monitoring/pom.xml | 2 - .../monitoring/client/Lwm2mClient.java | 133 +++++++++--------- 2 files changed, 69 insertions(+), 66 deletions(-) diff --git a/monitoring/pom.xml b/monitoring/pom.xml index 4f59cbc975..e17b87454e 100644 --- a/monitoring/pom.xml +++ b/monitoring/pom.xml @@ -42,8 +42,6 @@ ThingsBoard Monitoring Service org.thingsboard.monitoring.ThingsboardMonitoringApplication - 2.6.1 - 2.0.0-M4 diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/client/Lwm2mClient.java b/monitoring/src/main/java/org/thingsboard/monitoring/client/Lwm2mClient.java index f658bd1362..599fe29525 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/client/Lwm2mClient.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/client/Lwm2mClient.java @@ -20,13 +20,16 @@ import lombok.Setter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.eclipse.californium.core.network.CoapEndpoint; -import org.eclipse.californium.core.network.config.NetworkConfig; -import org.eclipse.californium.core.observe.ObservationStore; -import org.eclipse.californium.scandium.DTLSConnector; -import org.eclipse.californium.scandium.config.DtlsConnectorConfig; -import org.eclipse.leshan.client.californium.LeshanClient; -import org.eclipse.leshan.client.californium.LeshanClientBuilder; +import org.eclipse.californium.core.config.CoapConfig; +import org.eclipse.californium.elements.config.Configuration; +import org.eclipse.californium.scandium.config.DtlsConfig; +import org.eclipse.leshan.client.LeshanClient; +import org.eclipse.leshan.client.LeshanClientBuilder; +import org.eclipse.leshan.client.californium.endpoint.CaliforniumClientEndpointsProvider; +import org.eclipse.leshan.client.californium.endpoint.ClientProtocolProvider; +import org.eclipse.leshan.client.californium.endpoint.coap.CoapOscoreProtocolProvider; +import org.eclipse.leshan.client.californium.endpoint.coaps.CoapsClientProtocolProvider; +import org.eclipse.leshan.client.endpoint.LwM2mClientEndpointsProvider; import org.eclipse.leshan.client.engine.DefaultRegistrationEngineFactory; import org.eclipse.leshan.client.object.Security; import org.eclipse.leshan.client.object.Server; @@ -34,9 +37,8 @@ import org.eclipse.leshan.client.observer.LwM2mClientObserver; import org.eclipse.leshan.client.resource.BaseInstanceEnabler; import org.eclipse.leshan.client.resource.DummyInstanceEnabler; import org.eclipse.leshan.client.resource.ObjectsInitializer; -import org.eclipse.leshan.client.servers.ServerIdentity; +import org.eclipse.leshan.client.servers.LwM2mServer; import org.eclipse.leshan.core.ResponseCode; -import org.eclipse.leshan.core.californium.EndpointFactory; import org.eclipse.leshan.core.model.InvalidDDFFileException; import org.eclipse.leshan.core.model.LwM2mModel; import org.eclipse.leshan.core.model.ObjectLoader; @@ -53,7 +55,6 @@ import org.thingsboard.monitoring.util.ResourceUtils; import javax.security.auth.Destroyable; import java.io.IOException; -import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -95,9 +96,11 @@ public class Lwm2mClient extends BaseInstanceEnabler implements Destroyable { } Security security = noSec(serverUri, 123); - NetworkConfig coapConfig = new NetworkConfig().setString(NetworkConfig.Keys.COAP_PORT, StringUtils.substringAfterLast(serverUri, ":")); - - LeshanClient leshanClient; + Configuration coapConfig = new Configuration(); + String portStr = StringUtils.substringAfterLast(serverUri, ":"); + if (StringUtils.isNotEmpty(portStr)) { + coapConfig.set(CoapConfig.COAP_PORT, Integer.parseInt(portStr)); + } LwM2mModel model = new StaticModel(models); ObjectsInitializer initializer = new ObjectsInitializer(model); @@ -105,118 +108,121 @@ public class Lwm2mClient extends BaseInstanceEnabler implements Destroyable { initializer.setInstancesForObject(SERVER, new Server(123, TimeUnit.MINUTES.toSeconds(5))); initializer.setInstancesForObject(DEVICE, this); initializer.setClassForObject(ACCESS_CONTROL, DummyInstanceEnabler.class); - DtlsConnectorConfig.Builder dtlsConfig = new DtlsConnectorConfig.Builder(); - dtlsConfig.setRecommendedCipherSuitesOnly(true); - dtlsConfig.setClientOnly(); - DefaultRegistrationEngineFactory engineFactory = new DefaultRegistrationEngineFactory(); - engineFactory.setReconnectOnUpdate(false); - engineFactory.setResumeOnConnect(true); + // Create client endpoints Provider + List protocolProvider = new ArrayList<>(); + protocolProvider.add(new CoapOscoreProtocolProvider()); + protocolProvider.add(new CoapsClientProtocolProvider()); + CaliforniumClientEndpointsProvider.Builder endpointsBuilder = new CaliforniumClientEndpointsProvider.Builder( + protocolProvider.toArray(new ClientProtocolProvider[protocolProvider.size()])); - EndpointFactory endpointFactory = new EndpointFactory() { + // Create Californium Configuration + Configuration clientCoapConfig = endpointsBuilder.createDefaultConfiguration(); - @Override - public CoapEndpoint createUnsecuredEndpoint(InetSocketAddress address, NetworkConfig coapConfig, - ObservationStore store) { - CoapEndpoint.Builder builder = new CoapEndpoint.Builder(); - builder.setInetSocketAddress(address); - builder.setNetworkConfig(coapConfig); - return builder.build(); - } + // Set some DTLS stuff + clientCoapConfig.setTransient(DtlsConfig.DTLS_RECOMMENDED_CIPHER_SUITES_ONLY); + clientCoapConfig.set(DtlsConfig.DTLS_RECOMMENDED_CIPHER_SUITES_ONLY, true); - @Override - public CoapEndpoint createSecuredEndpoint(DtlsConnectorConfig dtlsConfig, NetworkConfig coapConfig, - ObservationStore store) { - CoapEndpoint.Builder builder = new CoapEndpoint.Builder(); - DtlsConnectorConfig.Builder dtlsConfigBuilder = new DtlsConnectorConfig.Builder(dtlsConfig); - builder.setConnector(new DTLSConnector(dtlsConfigBuilder.build())); - builder.setNetworkConfig(coapConfig); - return builder.build(); - } - }; + // Set Californium Configuration + endpointsBuilder.setConfiguration(clientCoapConfig); + + // creates EndpointsProvider + List endpointsProvider = new ArrayList<>(); + endpointsProvider.add(endpointsBuilder.build()); + // Configure registration engine + DefaultRegistrationEngineFactory engineFactory = new DefaultRegistrationEngineFactory(); + engineFactory.setReconnectOnUpdate(false); + engineFactory.setResumeOnConnect(true); + + // Build the client LeshanClientBuilder builder = new LeshanClientBuilder(endpoint); builder.setObjects(initializer.createAll()); - builder.setCoapConfig(coapConfig); - builder.setDtlsConfig(dtlsConfig); + builder.setEndpointsProviders(endpointsProvider.toArray(new LwM2mClientEndpointsProvider[endpointsProvider.size()])); builder.setRegistrationEngineFactory(engineFactory); - builder.setEndpointFactory(endpointFactory); builder.setDecoder(new DefaultLwM2mDecoder(false)); builder.setEncoder(new DefaultLwM2mEncoder(false)); leshanClient = builder.build(); + // Add observer LwM2mClientObserver observer = new LwM2mClientObserver() { - @Override - public void onBootstrapStarted(ServerIdentity bsserver, BootstrapRequest request) {} + public void onBootstrapStarted(LwM2mServer bsserver, BootstrapRequest request) { + // No implementation needed + } @Override - public void onBootstrapSuccess(ServerIdentity bsserver, BootstrapRequest request) {} + public void onBootstrapSuccess(LwM2mServer bsserver, BootstrapRequest request) { + // No implementation needed + } @Override - public void onBootstrapFailure(ServerIdentity bsserver, BootstrapRequest request, - ResponseCode responseCode, String errorMessage, Exception cause) {} + public void onBootstrapFailure(LwM2mServer bsserver, BootstrapRequest request, ResponseCode responseCode, String errorMessage, Exception cause) { + // No implementation needed + } @Override - public void onBootstrapTimeout(ServerIdentity bsserver, BootstrapRequest request) {} + public void onBootstrapTimeout(LwM2mServer bsserver, BootstrapRequest request) { + // No implementation needed + } @Override - public void onRegistrationStarted(ServerIdentity server, RegisterRequest request) { + public void onRegistrationStarted(LwM2mServer server, RegisterRequest request) { log.debug("onRegistrationStarted [{}]", request.getEndpointName()); } @Override - public void onRegistrationSuccess(ServerIdentity server, RegisterRequest request, String registrationID) { + public void onRegistrationSuccess(LwM2mServer server, RegisterRequest request, String registrationID) { log.debug("onRegistrationSuccess [{}] [{}]", request.getEndpointName(), registrationID); } @Override - public void onRegistrationFailure(ServerIdentity server, RegisterRequest request, ResponseCode responseCode, String errorMessage, Exception cause) { + public void onRegistrationFailure(LwM2mServer server, RegisterRequest request, ResponseCode responseCode, String errorMessage, Exception cause) { log.debug("onRegistrationFailure [{}] [{}] [{}]", request.getEndpointName(), responseCode, errorMessage); } @Override - public void onRegistrationTimeout(ServerIdentity server, RegisterRequest request) { + public void onRegistrationTimeout(LwM2mServer server, RegisterRequest request) { log.debug("onRegistrationTimeout [{}]", request.getEndpointName()); } @Override - public void onUpdateStarted(ServerIdentity server, UpdateRequest request) { + public void onUpdateStarted(LwM2mServer server, UpdateRequest request) { log.debug("onUpdateStarted [{}]", request.getRegistrationId()); } @Override - public void onUpdateSuccess(ServerIdentity server, UpdateRequest request) { + public void onUpdateSuccess(LwM2mServer server, UpdateRequest request) { log.debug("onUpdateSuccess [{}]", request.getRegistrationId()); } @Override - public void onUpdateFailure(ServerIdentity server, UpdateRequest request, ResponseCode responseCode, String errorMessage, Exception cause) { + public void onUpdateFailure(LwM2mServer server, UpdateRequest request, ResponseCode responseCode, String errorMessage, Exception cause) { log.debug("onUpdateFailure [{}]", request.getRegistrationId()); } @Override - public void onUpdateTimeout(ServerIdentity server, UpdateRequest request) { + public void onUpdateTimeout(LwM2mServer server, UpdateRequest request) { log.debug("onUpdateTimeout [{}]", request.getRegistrationId()); } @Override - public void onDeregistrationStarted(ServerIdentity server, DeregisterRequest request) { + public void onDeregistrationStarted(LwM2mServer server, DeregisterRequest request) { log.debug("onDeregistrationStarted [{}]", request.getRegistrationId()); } @Override - public void onDeregistrationSuccess(ServerIdentity server, DeregisterRequest request) { - log.debug("onDeregistrationStarted [{}]", request.getRegistrationId()); + public void onDeregistrationSuccess(LwM2mServer server, DeregisterRequest request) { + log.debug("onDeregistrationSuccess [{}]", request.getRegistrationId()); } @Override - public void onDeregistrationFailure(ServerIdentity server, DeregisterRequest request, ResponseCode responseCode, String errorMessage, Exception cause) { + public void onDeregistrationFailure(LwM2mServer server, DeregisterRequest request, ResponseCode responseCode, String errorMessage, Exception cause) { log.debug("onDeregistrationFailure [{}] [{}] [{}]", request.getRegistrationId(), responseCode, errorMessage); } @Override - public void onDeregistrationTimeout(ServerIdentity server, DeregisterRequest request) { + public void onDeregistrationTimeout(LwM2mServer server, DeregisterRequest request) { log.debug("onDeregistrationTimeout [{}]", request.getRegistrationId()); } @@ -224,7 +230,6 @@ public class Lwm2mClient extends BaseInstanceEnabler implements Destroyable { public void onUnexpectedError(Throwable unexpectedError) { log.debug("onUnexpectedError [{}]", unexpectedError.toString()); } - }; leshanClient.addObserver(observer); @@ -239,17 +244,17 @@ public class Lwm2mClient extends BaseInstanceEnabler implements Destroyable { } @Override - public ReadResponse read(ServerIdentity identity, int resourceId) { + public ReadResponse read(LwM2mServer server, int resourceId) { if (supportedResources.contains(resourceId)) { return ReadResponse.success(resourceId, data); } - return super.read(identity, resourceId); + return super.read(server, resourceId); } @SneakyThrows public void send(String data, int resource) { this.data = data; - fireResourcesChange(resource); + fireResourceChange(resource); } @Override From 14f24d24974c3c29f3b9f612de5fb0c8bd0e2a86 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 12 May 2025 11:28:16 +0300 Subject: [PATCH 43/79] Fix readiness status for local EDQS --- .../server/edqs/state/LocalEdqsStateService.java | 13 +++++++++---- .../server/queue/discovery/ZkDiscoveryService.java | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java index c9923e6194..00bed5e91d 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java @@ -29,6 +29,7 @@ import org.thingsboard.server.edqs.util.EdqsRocksDb; import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; +import org.thingsboard.server.queue.discovery.DiscoveryService; import org.thingsboard.server.queue.edqs.InMemoryEdqsComponent; import java.util.List; @@ -43,12 +44,14 @@ import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTop public class LocalEdqsStateService implements EdqsStateService { private final EdqsRocksDb db; + private final DiscoveryService discoveryService; @Autowired @Lazy private EdqsProcessor processor; private PartitionedQueueConsumerManager> eventConsumer; private List> otherConsumers; - private Set partitions; + + private boolean ready = false; @Override public void init(PartitionedQueueConsumerManager> eventConsumer, List> otherConsumers) { @@ -58,7 +61,7 @@ public class LocalEdqsStateService implements EdqsStateService { @Override public void process(Set partitions) { - if (this.partitions == null) { + if (!ready) { db.forEach((key, value) -> { try { ToEdqsMsg edqsMsg = ToEdqsMsg.parseFrom(value); @@ -70,11 +73,13 @@ public class LocalEdqsStateService implements EdqsStateService { }); log.info("Restore completed"); } + ready = true; + discoveryService.setReady(true); + eventConsumer.update(withTopic(partitions, eventConsumer.getTopic())); for (PartitionedQueueConsumerManager consumer : otherConsumers) { consumer.update(withTopic(partitions, consumer.getTopic())); } - this.partitions = partitions; } @Override @@ -93,7 +98,7 @@ public class LocalEdqsStateService implements EdqsStateService { @Override public boolean isReady() { - return partitions != null; + return ready; } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index 2d9ac3630e..d73f531b5b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -179,6 +179,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi @Override public void setReady(boolean ready) { + log.debug("Marking current service as {}", ready ? "ready" : "NOT ready"); boolean changed = serviceInfoProvider.setReady(ready); if (changed) { try { From afdfe451d5d6a48497e5a04e04a3399c98be4339 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 12 May 2025 12:50:47 +0200 Subject: [PATCH 44/79] test fix for givenSslIsTrueAndCredentials_whenGetSslContext_thenVerifySslContext --- .../java/org/thingsboard/rule/engine/mqtt/TbMqttNodeTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mqtt/TbMqttNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mqtt/TbMqttNodeTest.java index bf650af8bb..2438e59106 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mqtt/TbMqttNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/mqtt/TbMqttNodeTest.java @@ -181,7 +181,8 @@ public class TbMqttNodeTest extends AbstractRuleNodeUpgradeTest { SslContext actualSslContext = mqttClientConfig.getValue().getSslContext(); assertThat(actualSslContext) .usingRecursiveComparison() - .ignoringFields("ctx", "ctxLock", "sessionContext.context.ctx", "sessionContext.context.ctxLock") + .ignoringFields("ctx", "ctxLock", "sessionContext.context.ctx", "sessionContext.context.ctxLock", + "sslContext") .isEqualTo(SslContextBuilder.forClient().build()); } From be8149a4bc46c2b60ed949aaa5dba176ebe8b171 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Mon, 12 May 2025 14:25:24 +0300 Subject: [PATCH 45/79] UI: Fixed help icon color in rule node --- .../rule-node/action/gps-geo-action-config.component.html | 3 ++- .../action/save-to-custom-table-config.component.html | 1 + .../rule-node/action/timeseries-config.component.html | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/gps-geo-action-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/gps-geo-action-config.component.html index 2d3f230643..8849d60c77 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/gps-geo-action-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/gps-geo-action-config.component.html @@ -116,11 +116,12 @@ rule-node-config.polygon-definition - help + {{ 'rule-node-config.polygon-definition-required' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/save-to-custom-table-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/save-to-custom-table-config.component.html index c9685f2c3f..aee7b9cd83 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/save-to-custom-table-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/save-to-custom-table-config.component.html @@ -22,6 +22,7 @@ help diff --git a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html index 9a6d06255d..f41613c6fb 100644 --- a/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html +++ b/ui-ngx/src/app/modules/home/components/rule-node/action/timeseries-config.component.html @@ -76,9 +76,10 @@ requiredText="{{ 'rule-node-config.default-ttl-required' | translate }}" minErrorText="{{ 'rule-node-config.min-default-ttl-message' | translate }}" formControlName="defaultTTL"> - help From 40ed140952ff89051250d8b00b0d81b23d1eafe0 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 12 May 2025 15:18:38 +0300 Subject: [PATCH 46/79] Fix EDQS msa test --- .../java/org/thingsboard/server/msa/TestRestClient.java | 7 ++++--- .../server/msa/edqs/EdqsEntityDataQueryTest.java | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java index aafe93f9f3..102f4b82ea 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java @@ -42,6 +42,7 @@ import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.edqs.EdqsState; import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; @@ -738,13 +739,13 @@ public class TestRestClient { .as(Long.class); } - public Boolean isEdqsApiEnabled() { + public EdqsState getEdqsState() { return given().spec(requestSpec) - .get("/api/edqs/enabled") + .get("/api/edqs/state") .then() .statusCode(HTTP_OK) .extract() - .as(Boolean.class); + .as(EdqsState.class); } public void assignDeviceToCustomer(CustomerId customerId, DeviceId id) { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java index 53d8a72e7f..770d32e397 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java @@ -73,7 +73,7 @@ public class EdqsEntityDataQueryTest extends AbstractContainerTest { @BeforeClass public void beforeClass() throws Exception { testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); - await().atMost(60, TimeUnit.SECONDS).until(() -> testRestClient.isEdqsApiEnabled()); + await().atMost(60, TimeUnit.SECONDS).until(() -> testRestClient.getEdqsState().isApiEnabled()); tenantId = testRestClient.postTenant(EntityPrototypes.defaultTenantPrototype("Tenant")).getId(); tenantAdminId = testRestClient.createUserAndLogin(defaultTenantAdmin(tenantId, "tenantAdmin@thingsboard.org"), "tenant"); From d21b7fe0648ba5efb93c3aadc72b909ad5ae1744 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 12 May 2025 17:21:43 +0300 Subject: [PATCH 47/79] Ignore unknown properties for EdqsState --- .../java/org/thingsboard/server/common/data/edqs/EdqsState.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsState.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsState.java index ccd5406f93..52778ffd7e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsState.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsState.java @@ -15,12 +15,14 @@ */ package org.thingsboard.server.common.data.edqs; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Getter; import lombok.NoArgsConstructor; import org.apache.commons.lang3.BooleanUtils; @Getter @NoArgsConstructor +@JsonIgnoreProperties(ignoreUnknown = true) public class EdqsState { private Boolean edqsReady; From d15cb27ecf0df4e5c1caa0dcc5b6c7e0ba52e646 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 13 May 2025 15:19:24 +0300 Subject: [PATCH 48/79] Add queue prefix in msa tests --- .../test/java/org/thingsboard/server/msa/ContainerTestSuite.java | 1 + 1 file changed, 1 insertion(+) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index e00b0828c9..0573aee8fe 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -130,6 +130,7 @@ public class ContainerTestSuite { Map queueEnv = new HashMap<>(); queueEnv.put("TB_QUEUE_TYPE", QUEUE_TYPE); + queueEnv.put("TB_QUEUE_PREFIX", "test"); switch (QUEUE_TYPE) { case "kafka": composeFiles.add(new File(targetDir + "docker-compose.kafka.yml")); From ef65dd90263f982ff032ca835b91bd8d4e48e6a7 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 13 May 2025 16:41:57 +0300 Subject: [PATCH 49/79] Fix missing queue prefixes --- .../server/service/edqs/KafkaEdqsSyncService.java | 5 +++-- .../thingsboard/server/edqs/processor/EdqsProcessor.java | 8 +++++--- .../server/edqs/state/KafkaEdqsStateService.java | 6 ++++-- .../server/queue/discovery/HashPartitionService.java | 2 +- .../org/thingsboard/server/queue/kafka/TbKafkaAdmin.java | 3 +++ 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java index 239fd9dc42..43b0c575a0 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.edqs; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.edqs.EdqsConfig; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaSettings; @@ -32,11 +33,11 @@ public class KafkaEdqsSyncService extends EdqsSyncService { private final boolean syncNeeded; - public KafkaEdqsSyncService(TbKafkaSettings kafkaSettings, EdqsConfig edqsConfig) { + public KafkaEdqsSyncService(TbKafkaSettings kafkaSettings, TopicService topicService, EdqsConfig edqsConfig) { TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, Collections.emptyMap()); this.syncNeeded = kafkaAdmin.areAllTopicsEmpty(IntStream.range(0, edqsConfig.getPartitions()) .mapToObj(partition -> TopicPartitionInfo.builder() - .topic(edqsConfig.getEventsTopic()) + .topic(topicService.buildTopicName(edqsConfig.getEventsTopic())) .partition(partition) .build().getFullTopicName()) .collect(Collectors.toSet())); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java index 78d17ef368..72919c3fc0 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java @@ -58,6 +58,7 @@ import org.thingsboard.server.queue.TbQueueResponseTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.edqs.EdqsComponent; import org.thingsboard.server.queue.edqs.EdqsConfig; @@ -88,6 +89,7 @@ public class EdqsProcessor implements TbQueueHandler, private final EdqsRepository repository; private final EdqsConfig config; private final EdqsPartitionService partitionService; + private final TopicService topicService; private final ConfigurableApplicationContext applicationContext; private final EdqsStateService stateService; @@ -123,7 +125,7 @@ public class EdqsProcessor implements TbQueueHandler, eventConsumer = PartitionedQueueConsumerManager.>create() .queueKey(new QueueKey(ServiceType.EDQS, config.getEventsTopic())) - .topic(config.getEventsTopic()) + .topic(topicService.buildTopicName(config.getEventsTopic())) .pollInterval(config.getPollInterval()) .msgPackProcessor((msgs, consumer, config) -> { for (TbProtoQueueMsg queueMsg : msgs) { @@ -164,9 +166,9 @@ public class EdqsProcessor implements TbQueueHandler, try { Set newPartitions = event.getNewPartitions().get(new QueueKey(ServiceType.EDQS)); - stateService.process(withTopic(newPartitions, config.getStateTopic())); + stateService.process(withTopic(newPartitions, topicService.buildTopicName(config.getStateTopic()))); // eventsConsumer's partitions are updated by stateService - responseTemplate.subscribe(withTopic(newPartitions, config.getRequestsTopic())); // TODO: we subscribe to partitions before we are ready. implement consumer-per-partition version for request template + responseTemplate.subscribe(withTopic(newPartitions, topicService.buildTopicName(config.getRequestsTopic()))); // TODO: we subscribe to partitions before we are ready. implement consumer-per-partition version for request template Set oldPartitions = event.getOldPartitions().get(new QueueKey(ServiceType.EDQS)); if (CollectionsUtil.isNotEmpty(oldPartitions)) { diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java index 0efe6e7d3b..ddbdc3253a 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java @@ -36,6 +36,7 @@ import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; import org.thingsboard.server.queue.common.state.KafkaQueueStateService; import org.thingsboard.server.queue.common.state.QueueStateService; import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.edqs.EdqsConfig; import org.thingsboard.server.queue.edqs.KafkaEdqsComponent; import org.thingsboard.server.queue.edqs.KafkaEdqsQueueFactory; @@ -59,6 +60,7 @@ public class KafkaEdqsStateService implements EdqsStateService { private final EdqsConfig config; private final EdqsPartitionService partitionService; private final KafkaEdqsQueueFactory queueFactory; + private final TopicService topicService; @Autowired @Lazy private EdqsProcessor edqsProcessor; @@ -78,7 +80,7 @@ public class KafkaEdqsStateService implements EdqsStateService { TbKafkaAdmin queueAdmin = queueFactory.getEdqsQueueAdmin(); stateConsumer = PartitionedQueueConsumerManager.>create() .queueKey(new QueueKey(ServiceType.EDQS, config.getStateTopic())) - .topic(config.getStateTopic()) + .topic(topicService.buildTopicName(config.getStateTopic())) .pollInterval(config.getPollInterval()) .msgPackProcessor((msgs, consumer, config) -> { for (TbProtoQueueMsg queueMsg : msgs) { @@ -176,7 +178,7 @@ public class KafkaEdqsStateService implements EdqsStateService { if (queueStateService.getPartitions().isEmpty()) { Set allPartitions = IntStream.range(0, config.getPartitions()) .mapToObj(partition -> TopicPartitionInfo.builder() - .topic(config.getEventsTopic()) + .topic(topicService.buildTopicName(config.getEventsTopic())) .partition(partition) .build()) .collect(Collectors.toSet()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 7186bf7055..ec5c675e32 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -156,7 +156,7 @@ public class HashPartitionService implements PartitionService { @Override public String getTopic(QueueKey queueKey) { - return partitionTopicsMap.get(queueKey); + return topicService.buildTopicName(partitionTopicsMap.get(queueKey)); } private void doInitRuleEnginePartitions() { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java index 1e0064a5c8..6addce2865 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java @@ -136,6 +136,9 @@ public class TbKafkaAdmin implements TbQueueAdmin, TbEdgeQueueAdmin { } public CreateTopicsResult createTopic(NewTopic topic) { + if (!topic.name().startsWith("test.")) { // FIXME: remove me + log.error("Creating topic without configured prefix: {}", topic.name(), new RuntimeException("stacktrace")); + } return settings.getAdminClient().createTopics(Collections.singletonList(topic)); } From 64c1a1db0c5c7850262aaf4d1d72a4b9b4204aec Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 14 May 2025 09:14:54 +0300 Subject: [PATCH 50/79] Revert "test scope mockserver with no dependency to not affect transitive dependencies" This reverts commit d608b87f945337908b4c731a10116eab35b080b0. --- pom.xml | 4 ++-- rule-engine/rule-engine-components/pom.xml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 608a46584a..0260dc09be 100755 --- a/pom.xml +++ b/pom.xml @@ -2233,7 +2233,7 @@ org.mock-server - mockserver-netty-no-dependencies + mockserver-netty ${mock-server.version} test @@ -2245,7 +2245,7 @@ org.mock-server - mockserver-client-java-no-dependencies + mockserver-client-java ${mock-server.version} test diff --git a/rule-engine/rule-engine-components/pom.xml b/rule-engine/rule-engine-components/pom.xml index b975f7c624..008c422dbf 100644 --- a/rule-engine/rule-engine-components/pom.xml +++ b/rule-engine/rule-engine-components/pom.xml @@ -141,12 +141,12 @@ org.mock-server - mockserver-netty-no-dependencies + mockserver-netty test org.mock-server - mockserver-client-java-no-dependencies + mockserver-client-java test From 95d1b5ebd7de3f4eaa63ae8d2e9262912d5c05d9 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 15 May 2025 11:17:31 +0300 Subject: [PATCH 51/79] Minor refactoring for TopicPartitionInfo --- .../server/service/queue/DefaultTbCoreConsumerService.java | 2 +- .../server/common/msg/queue/TopicPartitionInfo.java | 4 ---- .../java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java | 3 --- .../queue/usagestats/DefaultTbApiUsageReportClient.java | 2 +- 4 files changed, 2 insertions(+), 9 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 47a0c473e9..d3743eb084 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -251,7 +251,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService tpi.newByTopic(usageStatsConsumer.getConsumer().getTopic())) + .map(tpi -> tpi.withTopic(usageStatsConsumer.getConsumer().getTopic())) .collect(Collectors.toSet())); } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java index b18debaf49..80eaede7bf 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java @@ -57,10 +57,6 @@ public class TopicPartitionInfo { this(topic, tenantId, partition, false, myPartition); } - public TopicPartitionInfo newByTopic(String topic) { - return new TopicPartitionInfo(topic, this.tenantId, this.partition, this.useInternalPartition, this.myPartition); - } - public String getTopic() { return topic; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java index 6addce2865..1e0064a5c8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java @@ -136,9 +136,6 @@ public class TbKafkaAdmin implements TbQueueAdmin, TbEdgeQueueAdmin { } public CreateTopicsResult createTopic(NewTopic topic) { - if (!topic.name().startsWith("test.")) { // FIXME: remove me - log.error("Creating topic without configured prefix: {}", topic.name(), new RuntimeException("stacktrace")); - } return settings.getAdminClient().createTopics(Collections.singletonList(topic)); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbApiUsageReportClient.java b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbApiUsageReportClient.java index 715020dc7c..34543a07e8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbApiUsageReportClient.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/usagestats/DefaultTbApiUsageReportClient.java @@ -131,7 +131,7 @@ public class DefaultTbApiUsageReportClient implements TbApiUsageReportClient { report.forEach((parent, statsMsg) -> { try { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, parent.getTenantId(), parent.getId()) - .newByTopic(msgProducer.getDefaultTopic()); + .withTopic(msgProducer.getDefaultTopic()); reportStatsPerTpi.computeIfAbsent(tpi, k -> new ArrayList<>()).add(statsMsg.build()); } catch (TenantNotFoundException e) { log.debug("Couldn't report usage stats for non-existing tenant: {}", e.getTenantId()); From 52f7683b322f3bfec1288343b9345b7d2a1785e1 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 15 May 2025 12:02:59 +0300 Subject: [PATCH 52/79] Cleanup ContainerTestSuite --- .../server/msa/ContainerTestSuite.java | 41 ++----------------- 1 file changed, 3 insertions(+), 38 deletions(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 0573aee8fe..5a53862164 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -29,7 +29,6 @@ import java.nio.file.Path; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; @@ -46,7 +45,6 @@ public class ContainerTestSuite { final static boolean IS_REDIS_SENTINEL = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisSentinel")); final static boolean IS_REDIS_SSL = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisSsl")); final static boolean IS_HYBRID_MODE = Boolean.parseBoolean(System.getProperty("blackBoxTests.hybridMode")); - final static String QUEUE_TYPE = System.getProperty("blackBoxTests.queue", "kafka"); private static final String SOURCE_DIR = "./../../docker/"; private static final String TB_CORE_LOG_REGEXP = ".*Starting polling for events.*"; private static final String TRANSPORTS_LOG_REGEXP = ".*Going to recalculate partitions.*"; @@ -122,45 +120,12 @@ public class ContainerTestSuite { new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid.yml" : "docker-compose.postgres.yml")), new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid-test-extras.yml" : "docker-compose.postgres-test-extras.yml")), new File(targetDir + "docker-compose.postgres.volumes.yml"), - new File(targetDir + "docker-compose." + QUEUE_TYPE + ".yml"), + new File(targetDir + "docker-compose.kafka.yml"), new File(targetDir + resolveRedisComposeFile()), new File(targetDir + resolveRedisComposeVolumesFile()), new File(targetDir + ("docker-selenium.yml")) )); - - Map queueEnv = new HashMap<>(); - queueEnv.put("TB_QUEUE_TYPE", QUEUE_TYPE); - queueEnv.put("TB_QUEUE_PREFIX", "test"); - switch (QUEUE_TYPE) { - case "kafka": - composeFiles.add(new File(targetDir + "docker-compose.kafka.yml")); - break; - case "aws-sqs": - replaceInFile(targetDir, "queue-aws-sqs.env", - Map.of("YOUR_KEY", getSysProp("blackBoxTests.awsKey"), - "YOUR_SECRET", getSysProp("blackBoxTests.awsSecret"), - "YOUR_REGION", getSysProp("blackBoxTests.awsRegion"))); - break; - case "rabbitmq": - composeFiles.add(new File(targetDir + "docker-compose.rabbitmq-server.yml")); - replaceInFile(targetDir, "queue-rabbitmq.env", - Map.of("localhost", "rabbitmq")); - break; - case "service-bus": - replaceInFile(targetDir, "queue-service-bus.env", - Map.of("YOUR_NAMESPACE_NAME", getSysProp("blackBoxTests.serviceBusNamespace"), - "YOUR_SAS_KEY_NAME", getSysProp("blackBoxTests.serviceBusSASPolicy"))); - replaceInFile(targetDir, "queue-service-bus.env", - Map.of("YOUR_SAS_KEY", getSysProp("blackBoxTests.serviceBusPrimaryKey"))); - break; - case "pubsub": - replaceInFile(targetDir, "queue-pubsub.env", - Map.of("YOUR_PROJECT_ID", getSysProp("blackBoxTests.pubSubProjectId"), - "YOUR_SERVICE_ACCOUNT", getSysProp("blackBoxTests.pubSubServiceAccount"))); - break; - default: - throw new RuntimeException("Unsupported queue type: " + QUEUE_TYPE); - } + addToFile(targetDir, "queue-kafka.env", Map.of("TB_QUEUE_PREFIX", "test")); if (IS_HYBRID_MODE) { composeFiles.add(new File(targetDir + "docker-compose.cassandra.volumes.yml")); @@ -172,7 +137,7 @@ public class ContainerTestSuite { .withOptions("--compatibility") .withTailChildContainers(!skipTailChildContainers) .withEnv(installTb.getEnv()) - .withEnv(queueEnv) + .withEnv("TB_QUEUE_TYPE", "kafka") .withEnv("LOAD_BALANCER_NAME", "") .withExposedService("haproxy", 80, Wait.forHttp("/swagger-ui.html").withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .withExposedService("broker", 1883) From d418e15b9f1569e6be7f5c1c88519494eebab4d2 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 19 May 2025 10:49:16 +0300 Subject: [PATCH 53/79] Add missing test queue prefix for EDQS --- .../test/java/org/thingsboard/server/msa/ContainerTestSuite.java | 1 + 1 file changed, 1 insertion(+) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 5a53862164..345d0b7144 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -126,6 +126,7 @@ public class ContainerTestSuite { new File(targetDir + ("docker-selenium.yml")) )); addToFile(targetDir, "queue-kafka.env", Map.of("TB_QUEUE_PREFIX", "test")); + addToFile(targetDir, "tb-edqs.env", Map.of("TB_QUEUE_PREFIX", "test")); if (IS_HYBRID_MODE) { composeFiles.add(new File(targetDir + "docker-compose.cassandra.volumes.yml")); From e12b4015b1f568d7b3de12072fd52a9bee19a0a3 Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Tue, 20 May 2025 12:06:30 +0300 Subject: [PATCH 54/79] UI: Fixed HP curcuit breaker widget type fqn --- .../widget_bundles/high_performance_scada_energy_system.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/data/json/system/widget_bundles/high_performance_scada_energy_system.json b/application/src/main/data/json/system/widget_bundles/high_performance_scada_energy_system.json index a06ccc0819..bc14e14023 100644 --- a/application/src/main/data/json/system/widget_bundles/high_performance_scada_energy_system.json +++ b/application/src/main/data/json/system/widget_bundles/high_performance_scada_energy_system.json @@ -15,7 +15,7 @@ "hp_wind_turbine_cluster", "hp_fuel_generator", "hp_industrial_fuel_generator", - "hp_circuit_breaker2", + "hp_circuit_breaker", "hp_horizontal_circuit_breaker", "hp_voltage_relay", "hp_3_phase_voltage_relay", From 440087384d3d478011a33756952d0d061c20af2f Mon Sep 17 00:00:00 2001 From: dshvaika Date: Tue, 20 May 2025 12:27:26 +0300 Subject: [PATCH 55/79] deduplication node: fixed retry mechanism --- .../deduplication/TbMsgDeduplicationNode.java | 17 +++-- .../transform/TbMsgDeduplicationNodeTest.java | 76 +++++++++++++++++++ 2 files changed, 85 insertions(+), 8 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/deduplication/TbMsgDeduplicationNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/deduplication/TbMsgDeduplicationNode.java index f7a7c6e4dd..ce5fba102a 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/deduplication/TbMsgDeduplicationNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/deduplication/TbMsgDeduplicationNode.java @@ -64,7 +64,7 @@ import static org.thingsboard.server.common.data.DataConstants.QUEUE_NAME; @Slf4j public class TbMsgDeduplicationNode implements TbNode { - public static final int TB_MSG_DEDUPLICATION_RETRY_DELAY = 10; + public static final long TB_MSG_DEDUPLICATION_RETRY_DELAY = 10L; private TbMsgDeduplicationNodeConfiguration config; @@ -217,16 +217,17 @@ public class TbMsgDeduplicationNode implements TbNode { } private void enqueueForTellNextWithRetry(TbContext ctx, TbMsg msg, int retryAttempt) { - if (config.getMaxRetries() > retryAttempt) { + if (retryAttempt <= config.getMaxRetries()) { ctx.enqueueForTellNext(msg, TbNodeConnectionType.SUCCESS, - () -> { - log.trace("[{}][{}][{}] Successfully enqueue deduplication result message!", ctx.getSelfId(), msg.getOriginator(), retryAttempt); - }, + () -> log.trace("[{}][{}][{}] Successfully enqueue deduplication result message!", ctx.getSelfId(), msg.getOriginator(), retryAttempt), throwable -> { log.trace("[{}][{}][{}] Failed to enqueue deduplication output message due to: ", ctx.getSelfId(), msg.getOriginator(), retryAttempt, throwable); - ctx.schedule(() -> { - enqueueForTellNextWithRetry(ctx, msg, retryAttempt + 1); - }, TB_MSG_DEDUPLICATION_RETRY_DELAY, TimeUnit.SECONDS); + if (retryAttempt < config.getMaxRetries()) { + ctx.schedule(() -> enqueueForTellNextWithRetry(ctx, msg, retryAttempt + 1), TB_MSG_DEDUPLICATION_RETRY_DELAY, TimeUnit.SECONDS); + } else { + log.trace("[{}][{}] Max retries [{}] exhausted. Dropping deduplication result message [{}]", + ctx.getSelfId(), msg.getOriginator(), config.getMaxRetries(), msg.getId()); + } }); } } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbMsgDeduplicationNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbMsgDeduplicationNodeTest.java index 4b9cc0aadd..1a0c3431d7 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbMsgDeduplicationNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbMsgDeduplicationNodeTest.java @@ -62,11 +62,13 @@ import java.util.function.Consumer; import java.util.stream.Stream; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -411,6 +413,80 @@ public class TbMsgDeduplicationNodeTest extends AbstractRuleNodeUpgradeTest { Assertions.assertEquals(msgWithLatestTsInSecondPack.getType(), actualMsg.getType()); } + @Test + public void given_maxRetriesIsZero_when_enqueueFails_then_noRetriesIsScheduled() throws TbNodeException, ExecutionException, InterruptedException { + int wantedNumberOfTellSelfInvocation = 1; + int msgCount = 1; + awaitTellSelfLatch = new CountDownLatch(wantedNumberOfTellSelfInvocation); + invokeTellSelf(wantedNumberOfTellSelfInvocation); + + // Given + when(ctx.getQueueName()).thenReturn(DataConstants.MAIN_QUEUE_NAME); + config.setInterval(deduplicationInterval); + config.setStrategy(DeduplicationStrategy.FIRST); + config.setMaxPendingMsgs(msgCount); + config.setMaxRetries(0); + nodeConfiguration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); + node.init(ctx, nodeConfiguration); + + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + long currentTimeMillis = System.currentTimeMillis(); + + doAnswer(invocation -> { + Consumer failureCallback = invocation.getArgument(3); + failureCallback.accept(new RuntimeException("Simulated failure")); + return null; + }).when(ctx).enqueueForTellNext(any(), eq(TbNodeConnectionType.SUCCESS), any(), any()); + + TbMsg msg = createMsg(deviceId, currentTimeMillis + 1); + node.onMsg(ctx, msg); + + awaitTellSelfLatch.await(); + + verify(ctx).enqueueForTellNext(any(), eq(TbNodeConnectionType.SUCCESS), any(), any()); + verify(ctx, never()).schedule(any(), anyLong(), any()); + } + + @Test + public void given_maxRetriesIsSetToOne_when_enqueueFails_then_onlyOneRetryIsScheduled() throws TbNodeException, ExecutionException, InterruptedException { + int wantedNumberOfTellSelfInvocation = 1; + int msgCount = 1; + awaitTellSelfLatch = new CountDownLatch(wantedNumberOfTellSelfInvocation); + invokeTellSelf(wantedNumberOfTellSelfInvocation); + + when(ctx.getQueueName()).thenReturn(DataConstants.MAIN_QUEUE_NAME); + config.setInterval(deduplicationInterval); + config.setStrategy(DeduplicationStrategy.FIRST); + config.setMaxPendingMsgs(msgCount); + config.setMaxRetries(1); + nodeConfiguration = new TbNodeConfiguration(JacksonUtil.valueToTree(config)); + node.init(ctx, nodeConfiguration); + + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + long currentTimeMillis = System.currentTimeMillis(); + + doAnswer(invocation -> { + Consumer failureCallback = invocation.getArgument(3); + failureCallback.accept(new RuntimeException("Simulated failure")); + return null; + }).when(ctx).enqueueForTellNext(any(), eq(TbNodeConnectionType.SUCCESS), any(), any()); + + TbMsg msg = createMsg(deviceId, currentTimeMillis + 1); + node.onMsg(ctx, msg); + + awaitTellSelfLatch.await(); + + ArgumentCaptor retryRunnableCaptor = ArgumentCaptor.forClass(Runnable.class); + verify(ctx).schedule(retryRunnableCaptor.capture(), eq(TbMsgDeduplicationNode.TB_MSG_DEDUPLICATION_RETRY_DELAY), eq(TimeUnit.SECONDS)); + + retryRunnableCaptor.getValue().run(); + + // Verify total enqueue attempts (initial + retry) + verify(ctx, times(2)).enqueueForTellNext(any(), eq(TbNodeConnectionType.SUCCESS), any(), any()); + // No more retries scheduled after reaching maxRetries + verify(ctx).schedule(any(), eq(TbMsgDeduplicationNode.TB_MSG_DEDUPLICATION_RETRY_DELAY), eq(TimeUnit.SECONDS)); + } + // Rule nodes upgrade private static Stream givenFromVersionAndConfig_whenUpgrade_thenVerifyHasChangesAndConfig() { return Stream.of( From ebf41af1a231005f3796800934edf8472186bd00 Mon Sep 17 00:00:00 2001 From: yevhenii Date: Tue, 20 May 2025 18:22:24 +0300 Subject: [PATCH 56/79] Update edge install instructions - Minor instruction adjustments --- .../edge/instructions/install/centos/instructions.md | 4 +++- .../edge/instructions/install/docker/instructions.md | 11 +++++++---- .../edge/instructions/install/ubuntu/instructions.md | 6 ++++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/application/src/main/data/json/edge/instructions/install/centos/instructions.md b/application/src/main/data/json/edge/instructions/install/centos/instructions.md index 90d6d6ddc8..de5e57d926 100644 --- a/application/src/main/data/json/edge/instructions/install/centos/instructions.md +++ b/application/src/main/data/json/edge/instructions/install/centos/instructions.md @@ -41,6 +41,8 @@ OpenJDK 64-Bit Server VM (build ...) #### Step 2. Configure ThingsBoard Database ThingsBoard Edge supports SQL and hybrid database approaches. +In this guide we will use SQL only. +For hybrid details please follow official installation instructions from the ThingsBoard documentation site. ### PostgresSql ThingsBoard Edge uses PostgreSQL database as a local storage. @@ -111,7 +113,7 @@ sudo systemctl restart postgresql-16.service && psql -U postgres -d postgres -h #### Step 3. Choose Queue Service -ThingsBoard Edge is able to use different messaging systems/brokers for storing the messages and communication between ThingsBoard services. +ThingsBoard Edge supports only Kafka or in-memory queue (since v4.0) for message storage and communication between ThingsBoard services. How to choose the right queue implementation? In Memory queue implementation is built-in and default. It is useful for development(PoC) environments and is not suitable for production deployments or any sort of cluster deployments. diff --git a/application/src/main/data/json/edge/instructions/install/docker/instructions.md b/application/src/main/data/json/edge/instructions/install/docker/instructions.md index 3d42cffa8a..be052d9e8a 100644 --- a/application/src/main/data/json/edge/instructions/install/docker/instructions.md +++ b/application/src/main/data/json/edge/instructions/install/docker/instructions.md @@ -12,15 +12,18 @@ Here you can find ThingsBoard Edge docker image: #### Step 2. Choose Queue and/or Database Services -ThingsBoard Edge is able to use different messaging systems/brokers for storing the messages and communication between ThingsBoard services. +ThingsBoard Edge supports only Kafka or in-memory queue (since v4.0) for message storage and communication between ThingsBoard services. + +ThingsBoard Edge supports SQL and hybrid database approaches. +In this guide we will use SQL only. +For hybrid details please follow official installation instructions from the ThingsBoard documentation site. + How to choose the right queue implementation? In Memory queue implementation is built-in and default. It is useful for development(PoC) environments and is not suitable for production deployments or any sort of cluster deployments. Kafka is recommended for production deployments. This queue is used on the most of ThingsBoard production environments now. -Hybrid implementation combines PostgreSQL and Cassandra databases with Kafka queue service. It is recommended if you plan to manage 1M+ devices in production or handle high data ingestion rate (more than 5000 msg/sec). - Create a docker compose file for the ThingsBoard Edge service: ##### In Memory @@ -54,7 +57,7 @@ services: - tb-edge-logs:/var/log/tb-edge postgres: restart: always - image: "postgres:15" + image: "postgres:16" ports: - "5432" environment: diff --git a/application/src/main/data/json/edge/instructions/install/ubuntu/instructions.md b/application/src/main/data/json/edge/instructions/install/ubuntu/instructions.md index 6ac45d325a..a56b83e558 100644 --- a/application/src/main/data/json/edge/instructions/install/ubuntu/instructions.md +++ b/application/src/main/data/json/edge/instructions/install/ubuntu/instructions.md @@ -32,7 +32,9 @@ OpenJDK 64-Bit Server VM (...) #### Step 2. Configure ThingsBoard Edge Database -ThingsBoard Edge supports SQL and hybrid database approaches. See the architecture page for details. +ThingsBoard Edge supports SQL and hybrid database approaches. +In this guide we will use SQL only. +For hybrid details please follow official installation instructions from the ThingsBoard documentation site. ### Configure PostgreSQL ThingsBoard Edge uses PostgreSQL database as a local storage. @@ -71,7 +73,7 @@ echo "CREATE DATABASE tb_edge;" | psql -U postgres -d postgres -h 127.0.0.1 -W #### Step 3. Choose Queue Service -ThingsBoard Edge can use different messaging systems and brokers for storing messages and enabling communication between its services. Choose the appropriate queue implementation based on your specific business needs: +ThingsBoard Edge supports only Kafka or in-memory queue (since v4.0) for message storage and communication between ThingsBoard services. Choose the appropriate queue implementation based on your specific business needs: In Memory: The built-in and default queue implementation. It is useful for development or proof-of-concept (PoC) environments, but is not recommended for production or any type of clustered deployments due to limited scalability. From 7c7886993f95fb9c0264e2ac60b06a6ae3950a2e Mon Sep 17 00:00:00 2001 From: yevhenii Date: Tue, 20 May 2025 18:27:12 +0300 Subject: [PATCH 57/79] Update edge install instructions - Minor instruction adjustments --- .../data/json/edge/instructions/install/docker/instructions.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/application/src/main/data/json/edge/instructions/install/docker/instructions.md b/application/src/main/data/json/edge/instructions/install/docker/instructions.md index be052d9e8a..e568d273af 100644 --- a/application/src/main/data/json/edge/instructions/install/docker/instructions.md +++ b/application/src/main/data/json/edge/instructions/install/docker/instructions.md @@ -24,6 +24,8 @@ In Memory queue implementation is built-in and default. It is useful for develop Kafka is recommended for production deployments. This queue is used on the most of ThingsBoard production environments now. +Hybrid implementation combines PostgreSQL and Cassandra databases with Kafka queue service. It is recommended if you plan to manage 1M+ devices in production or handle high data ingestion rate (more than 5000 msg/sec). + Create a docker compose file for the ThingsBoard Edge service: ##### In Memory From 48c1842d420650dafbc584635d0831af9e5005b2 Mon Sep 17 00:00:00 2001 From: Yevhenii Date: Wed, 21 May 2025 12:56:23 +0300 Subject: [PATCH 58/79] Add validation for DefaultEdgeRuleChain in DeviceProfile - Refactored parameter name for clarity --- .../dao/service/validator/DeviceProfileDataValidator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java index e32195022c..dfd0ee82bc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java @@ -209,12 +209,12 @@ public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator Date: Fri, 23 May 2025 15:00:35 +0300 Subject: [PATCH 59/79] Merge pull request #13075 from jekka001/fix-sorting-edge-event Fix sorting edge event --- .../service/edge/rpc/EdgeGrpcSession.java | 2 +- .../rpc/fetch/GeneralEdgeEventFetcher.java | 6 ++- .../server/controller/EdgeControllerTest.java | 48 ++++++++++++------- .../server/edge/AbstractEdgeTest.java | 21 ++++++-- .../server/edge/DeviceProfileEdgeTest.java | 12 +++-- .../server/edge/TenantProfileEdgeTest.java | 9 +++- .../server/edge/imitator/EdgeImitator.java | 15 +++++- .../dao/sql/edge/JpaBaseEdgeEventDao.java | 10 ++-- .../dao/service/EdgeEventServiceTest.java | 25 ++++------ 9 files changed, 98 insertions(+), 50 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index 7861885989..4a9b68fc6d 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -712,7 +712,7 @@ public abstract class EdgeGrpcSession implements Closeable { private long findStartSeqIdFromOldestEventIfAny() { long startSeqId = 0L; try { - TimePageLink pageLink = new TimePageLink(1, 0, null, new SortOrder("createdTime"), null, null); + TimePageLink pageLink = new TimePageLink(1, 0, null, null, null, null); PageData edgeEvents = ctx.getEdgeEventService().findEdgeEvents(edge.getTenantId(), edge.getId(), null, null, pageLink); if (!edgeEvents.getData().isEmpty()) { startSeqId = edgeEvents.getData().get(0).getSeqId() - 1; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/GeneralEdgeEventFetcher.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/GeneralEdgeEventFetcher.java index 0515f23b16..5d7df601b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/GeneralEdgeEventFetcher.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/GeneralEdgeEventFetcher.java @@ -26,9 +26,13 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.edge.EdgeEventService; +import java.util.concurrent.TimeUnit; + @AllArgsConstructor @Slf4j public class GeneralEdgeEventFetcher implements EdgeEventFetcher { + // Subtract from queueStartTs to ensure no data is lost due to potential misordering of edge events by created_time. + private static final long MISORDERING_COMPENSATION_MILLIS = TimeUnit.SECONDS.toMillis(60); private final Long queueStartTs; private Long seqIdStart; @@ -44,7 +48,7 @@ public class GeneralEdgeEventFetcher implements EdgeEventFetcher { 0, null, null, - queueStartTs, + queueStartTs > 0 ? queueStartTs - MISORDERING_COMPENSATION_MILLIS : 0, System.currentTimeMillis()); } diff --git a/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java index 27c7da09b2..faefab4cb4 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java @@ -60,6 +60,7 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; @@ -68,6 +69,7 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.security.model.JwtSettings; import org.thingsboard.server.dao.edge.EdgeDao; import org.thingsboard.server.dao.exception.DataValidationException; @@ -107,6 +109,7 @@ import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.containsString; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; +import static org.thingsboard.server.edge.AbstractEdgeTest.CONNECT_MESSAGE_COUNT; @TestPropertySource(properties = { "edges.enabled=true", @@ -138,6 +141,7 @@ public class EdgeControllerTest extends AbstractControllerTest { public EdgeDao edgeDao(EdgeDao edgeDao) { return Mockito.mock(EdgeDao.class, AdditionalAnswers.delegatesTo(edgeDao)); } + } @Before @@ -886,6 +890,8 @@ public class EdgeControllerTest extends AbstractControllerTest { Device savedDevice = doPost("/api/device", device, Device.class); // create public customer + //1 message + // Customer doPost("/api/customer/public/device/" + savedDevice.getId().getId(), Device.class); doDelete("/api/customer/device/" + savedDevice.getId().getId(), Device.class); @@ -897,13 +903,16 @@ public class EdgeControllerTest extends AbstractControllerTest { + "/asset/" + savedAsset.getId().getId().toString(), Asset.class); EdgeImitator edgeImitator = new EdgeImitator(EDGE_HOST, EDGE_PORT, edge.getRoutingKey(), edge.getSecret()); - edgeImitator.ignoreType(UserCredentialsUpdateMsg.class); edgeImitator.ignoreType(OAuth2ClientUpdateMsg.class); edgeImitator.ignoreType(OAuth2DomainUpdateMsg.class); - edgeImitator.expectMessageAmount(27); + // 17 connect message + // + 1 Customer + // + 5 fetchers messages (DeviceProfile, Device, DeviceCredentials, AssetProfile, Asset) in sync process + // + 5 queue messages the same + edgeImitator.expectMessageAmount(CONNECT_MESSAGE_COUNT + 11); edgeImitator.connect(); - waitForMessages(edgeImitator); + edgeImitator.waitForMessages(); verifyFetchersMsgs(edgeImitator, savedDevice); // verify queue msgs @@ -914,9 +923,12 @@ public class EdgeControllerTest extends AbstractControllerTest { Assert.assertTrue(popAssetMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "Test Sync Edge Asset 1")); printQueueMsgsIfNotEmpty(edgeImitator); - edgeImitator.expectMessageAmount(21); + // 17 connect messages + // + 1 Customer + // + 5 fetchers messages (DeviceProfile, Device, DeviceCredentials, AssetProfile, Asset) in sync process + edgeImitator.expectMessageAmount(CONNECT_MESSAGE_COUNT + 6); doPost("/api/edge/sync/" + edge.getId()).andExpect(status().isOk()); - waitForMessages(edgeImitator); + edgeImitator.waitForMessages(); verifyFetchersMsgs(edgeImitator, savedDevice); printQueueMsgsIfNotEmpty(edgeImitator); @@ -987,17 +999,6 @@ public class EdgeControllerTest extends AbstractControllerTest { }); } - private void waitForMessages(EdgeImitator edgeImitator) throws Exception { - boolean success = edgeImitator.waitForMessages(); - if (!success) { - List downlinkMsgs = edgeImitator.getDownlinkMsgs(); - for (AbstractMessage downlinkMsg : downlinkMsgs) { - log.error("{}\n{}", downlinkMsg.getClass(), downlinkMsg); - } - Assert.fail("Await for messages was not successful!"); - } - } - private void verifyFetchersMsgs(EdgeImitator edgeImitator, Device savedDevice) { Assert.assertTrue(popQueueMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "Main")); Assert.assertTrue(popRuleChainMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "Edge Root Rule Chain")); @@ -1011,6 +1012,7 @@ public class EdgeControllerTest extends AbstractControllerTest { Assert.assertTrue(popAssetProfileMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "default")); Assert.assertTrue(popDeviceProfileMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "default")); Assert.assertTrue(popAssetProfileMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "default")); + Assert.assertTrue(popUserCredentialsMsg(edgeImitator.getDownlinkMsgs(), currentUserId)); Assert.assertTrue(popUserMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, TENANT_ADMIN_EMAIL, Authority.TENANT_ADMIN)); Assert.assertTrue(popCustomerMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "Public")); Assert.assertTrue(popDeviceProfileMsg(edgeImitator.getDownlinkMsgs(), UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, "default")); @@ -1156,6 +1158,20 @@ public class EdgeControllerTest extends AbstractControllerTest { return false; } + private boolean popUserCredentialsMsg(List messages, UserId userId) { + for (AbstractMessage message : messages) { + if (message instanceof UserCredentialsUpdateMsg userCredentialsUpdateMsg) { + UserCredentials userCredentials = JacksonUtil.fromString(userCredentialsUpdateMsg.getEntity(), UserCredentials.class, true); + Assert.assertNotNull(userCredentials); + if (userId.equals(userCredentials.getUserId())) { + messages.remove(message); + return true; + } + } + } + return false; + } + private boolean popUserMsg(List messages, UpdateMsgType msgType, String email, Authority authority) { for (AbstractMessage message : messages) { if (message instanceof UserUpdateMsg userUpdateMsg) { diff --git a/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java index d84e0ebea0..c02e72a35f 100644 --- a/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java @@ -115,7 +115,9 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. }) @Slf4j abstract public class AbstractEdgeTest extends AbstractControllerTest { - + public static final Integer CONNECT_MESSAGE_COUNT = 17; + public static final Integer INSTALLATION_MESSAGE_COUNT = 8; + public static final Integer SYNC_MESSAGE_COUNT = CONNECT_MESSAGE_COUNT + INSTALLATION_MESSAGE_COUNT; private static final String THERMOSTAT_DEVICE_PROFILE_NAME = "Thermostat"; protected DeviceProfile thermostatDeviceProfile; @@ -136,11 +138,12 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { doPost("/api/admin/jwtSettings", settings).andExpect(status().isOk()); loginTenantAdmin(); - + //8 installation messages installation(); edgeImitator = new EdgeImitator("localhost", 7070, edge.getRoutingKey(), edge.getSecret()); - edgeImitator.expectMessageAmount(25); + // 17 connect messages + 8 installation messages + edgeImitator.expectMessageAmount(SYNC_MESSAGE_COUNT); edgeImitator.ignoreType(OAuth2ClientUpdateMsg.class); edgeImitator.ignoreType(OAuth2DomainUpdateMsg.class); edgeImitator.connect(); @@ -164,22 +167,32 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { thermostatDeviceProfile = this.createDeviceProfile(THERMOSTAT_DEVICE_PROFILE_NAME, createMqttDeviceProfileTransportConfiguration(new JsonTransportPayloadConfiguration(), false)); extendDeviceProfileData(thermostatDeviceProfile); + //2 messages DeviceProfile thermostatDeviceProfile = doPost("/api/deviceProfile", thermostatDeviceProfile, DeviceProfile.class); Device savedDevice = saveDevice("Edge Device 1", THERMOSTAT_DEVICE_PROFILE_NAME); // create public customer + //1 message + // Customer doPost("/api/customer/public/device/" + savedDevice.getId().getId(), Device.class); doDelete("/api/customer/device/" + savedDevice.getId().getId(), Device.class); - Asset savedAsset = saveAsset("Edge Asset 1"); + Asset savedAsset = saveAsset("Edge Asset 1"); updateRootRuleChainMetadata(); edge = doPost("/api/edge", constructEdge("Test Edge", "test"), Edge.class); + //3 messages + // Device + // DeviceProfile + // DeviceCredentials doPost("/api/edge/" + edge.getUuidId() + "/device/" + savedDevice.getUuidId(), Device.class); + //2 messages + // Asset + // AssetProfile doPost("/api/edge/" + edge.getUuidId() + "/asset/" + savedAsset.getUuidId(), Asset.class); diff --git a/application/src/test/java/org/thingsboard/server/edge/DeviceProfileEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/DeviceProfileEdgeTest.java index 819dec5c0d..a29d7a4672 100644 --- a/application/src/test/java/org/thingsboard/server/edge/DeviceProfileEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/DeviceProfileEdgeTest.java @@ -128,15 +128,19 @@ public class DeviceProfileEdgeTest extends AbstractEdgeTest { @Test public void testDeleteDeviceProfilesWhenEdgeIsOffline() throws Exception { + //2 message RuleChain and RuleChainMetadata RuleChainId thermostatsRuleChainId = createEdgeRuleChainAndAssignToEdge("Thermostats Rule Chain"); // create device profile DeviceProfile deviceProfile = this.createDeviceProfile("ONE_MORE_DEVICE_PROFILE", null); deviceProfile.setDefaultEdgeRuleChainId(thermostatsRuleChainId); extendDeviceProfileData(deviceProfile); + + //1 message DeviceProfile edgeImitator.expectMessageAmount(1); deviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); Assert.assertTrue(edgeImitator.waitForMessages()); + AbstractMessage latestMessage = edgeImitator.getLatestMessage(); Assert.assertTrue(latestMessage instanceof DeviceProfileUpdateMsg); DeviceProfileUpdateMsg deviceProfileUpdateMsg = (DeviceProfileUpdateMsg) latestMessage; @@ -150,9 +154,11 @@ public class DeviceProfileEdgeTest extends AbstractEdgeTest { doDelete("/api/deviceProfile/" + deviceProfile.getUuidId()) .andExpect(status().isOk()); edgeImitator.connect(); - // 27 sync message - // + 1 delete message - edgeImitator.expectMessageAmount(28); + + // 25 sync message + // + 2 RuleChain and RuleChainMetadata + // + 1 delete DeviceProfile + edgeImitator.expectMessageAmount(SYNC_MESSAGE_COUNT + 3); Assert.assertTrue(edgeImitator.waitForMessages()); latestMessage = edgeImitator.getLatestMessage(); diff --git a/application/src/test/java/org/thingsboard/server/edge/TenantProfileEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/TenantProfileEdgeTest.java index 51cfa44ff9..c7e9ed1575 100644 --- a/application/src/test/java/org/thingsboard/server/edge/TenantProfileEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/TenantProfileEdgeTest.java @@ -78,11 +78,14 @@ public class TenantProfileEdgeTest extends AbstractEdgeTest { TenantProfileQueueConfiguration mainQueueConfiguration = createQueueConfig(DataConstants.MAIN_QUEUE_NAME, DataConstants.MAIN_QUEUE_TOPIC); TenantProfileQueueConfiguration isolatedQueueConfiguration = createQueueConfig("IsolatedHighPriority", "tb_rule_engine.isolated_hp"); edgeTenantProfile.getProfileData().setQueueConfiguration(List.of(mainQueueConfiguration, isolatedQueueConfiguration)); + // + 1 TenantProfile + // + 1 Queue main + // + 1 Queue isolated edgeImitator.expectMessageAmount(3); edgeTenantProfile = doPost("/api/tenantProfile", edgeTenantProfile, TenantProfile.class); Assert.assertTrue(edgeImitator.waitForMessages()); - Optional tenantProfileUpdateMsgOpt = edgeImitator.findMessageByType(TenantProfileUpdateMsg.class); + Optional tenantProfileUpdateMsgOpt = edgeImitator.findMessageByType(TenantProfileUpdateMsg.class); Assert.assertTrue(tenantProfileUpdateMsgOpt.isPresent()); TenantProfileUpdateMsg tenantProfileUpdateMsg = tenantProfileUpdateMsgOpt.get(); TenantProfile tenantProfile = JacksonUtil.fromString(tenantProfileUpdateMsg.getEntity(), TenantProfile.class, true); @@ -96,7 +99,9 @@ public class TenantProfileEdgeTest extends AbstractEdgeTest { loginTenantAdmin(); - edgeImitator.expectMessageAmount(21); + // 25 sync message + // +1 isolated Queue + edgeImitator.expectMessageAmount(SYNC_MESSAGE_COUNT + 1); doPost("/api/edge/sync/" + edge.getId()); assertThat(edgeImitator.waitForMessages()).as("await for messages after edge sync rest api call").isTrue(); diff --git a/application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java b/application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java index be9492bc53..8b259cf8fc 100644 --- a/application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java +++ b/application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java @@ -24,6 +24,7 @@ import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.checkerframework.checker.nullness.qual.Nullable; +import org.junit.Assert; import org.thingsboard.edge.rpc.EdgeGrpcClient; import org.thingsboard.edge.rpc.EdgeRpcClient; import org.thingsboard.server.controller.AbstractWebTest; @@ -386,7 +387,19 @@ public class EdgeImitator { } public boolean waitForMessages() throws InterruptedException { - return waitForMessages(AbstractWebTest.TIMEOUT); + boolean success = waitForMessages(AbstractWebTest.TIMEOUT); + + if (!success) { + List downlinkMsgs = getDownlinkMsgs(); + for (AbstractMessage downlinkMsg : downlinkMsgs) { + log.error("{}\n{}", downlinkMsg.getClass(), downlinkMsg); + } + + log.error("message count: {}", downlinkMsgs.size()); + Assert.fail("Await for messages was not successful!"); + } + + return true; } public boolean waitForMessages(int timeoutInSeconds) throws InterruptedException { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java index 4b6b898f42..ea2abbba8a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java @@ -44,7 +44,7 @@ import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; import org.thingsboard.server.dao.util.SqlDao; -import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.UUID; @@ -58,6 +58,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @RequiredArgsConstructor @Slf4j public class JpaBaseEdgeEventDao extends JpaPartitionedAbstractDao implements EdgeEventDao { + private static final List SORT_ORDERS = Collections.singletonList(new SortOrder("seqId")); private final UUID systemTenantId = NULL_UUID; @@ -175,11 +176,6 @@ public class JpaBaseEdgeEventDao extends JpaPartitionedAbstractDao findEdgeEvents(UUID tenantId, EdgeId edgeId, Long seqIdStart, Long seqIdEnd, TimePageLink pageLink) { - List sortOrders = new ArrayList<>(); - if (pageLink.getSortOrder() != null) { - sortOrders.add(pageLink.getSortOrder()); - } - sortOrders.add(new SortOrder("seqId")); return DaoUtil.toPageData( edgeEventRepository .findEdgeEventsByTenantIdAndEdgeId( @@ -190,7 +186,7 @@ public class JpaBaseEdgeEventDao extends JpaPartitionedAbstractDao> futures = new ArrayList<>(); - futures.add(saveEdgeEventWithProvidedTime(timeBeforeStartTime, edgeId, deviceId, tenantId)); - futures.add(saveEdgeEventWithProvidedTime(eventTime, edgeId, deviceId, tenantId)); - futures.add(saveEdgeEventWithProvidedTime(eventTime + 1, edgeId, deviceId, tenantId)); - futures.add(saveEdgeEventWithProvidedTime(eventTime + 2, edgeId, deviceId, tenantId)); - futures.add(saveEdgeEventWithProvidedTime(timeAfterEndTime, edgeId, deviceId, tenantId)); - - Futures.allAsList(futures).get(); + saveEdgeEventWithProvidedTime(timeBeforeStartTime, edgeId, deviceId, tenantId).get(); + saveEdgeEventWithProvidedTime(eventTime, edgeId, deviceId, tenantId).get(); + saveEdgeEventWithProvidedTime(eventTime + 2, edgeId, deviceId, tenantId).get(); + saveEdgeEventWithProvidedTime(eventTime + 1, edgeId, deviceId, tenantId).get(); + saveEdgeEventWithProvidedTime(timeAfterEndTime, edgeId, deviceId, tenantId).get(); TimePageLink pageLink = new TimePageLink(2, 0, "", new SortOrder("createdTime", SortOrder.Direction.DESC), startTime, endTime); PageData edgeEvents = edgeEventService.findEdgeEvents(tenantId, edgeId, 0L, null, pageLink); Assert.assertNotNull(edgeEvents.getData()); Assert.assertEquals(2, edgeEvents.getData().size()); - Assert.assertEquals(Uuids.startOf(eventTime + 2), edgeEvents.getData().get(0).getUuidId()); - Assert.assertEquals(Uuids.startOf(eventTime + 1), edgeEvents.getData().get(1).getUuidId()); + Assert.assertEquals(Uuids.startOf(eventTime), edgeEvents.getData().get(0).getUuidId()); + Assert.assertEquals(Uuids.startOf(eventTime + 2), edgeEvents.getData().get(1).getUuidId()); Assert.assertTrue(edgeEvents.hasNext()); Assert.assertNotNull(pageLink.nextPageLink()); @@ -130,7 +124,7 @@ public class EdgeEventServiceTest extends AbstractServiceTest { Assert.assertNotNull(edgeEvents.getData()); Assert.assertEquals(1, edgeEvents.getData().size()); - Assert.assertEquals(Uuids.startOf(eventTime), edgeEvents.getData().get(0).getUuidId()); + Assert.assertEquals(Uuids.startOf(eventTime + 1), edgeEvents.getData().get(0).getUuidId()); Assert.assertFalse(edgeEvents.hasNext()); edgeEventDao.cleanupEvents(1); @@ -141,4 +135,5 @@ public class EdgeEventServiceTest extends AbstractServiceTest { edgeEvent.setId(new EdgeEventId(Uuids.startOf(time))); return edgeEventService.saveAsync(edgeEvent); } + } \ No newline at end of file From ae4bdb52da5b82056431e63662661b08f06fcfb5 Mon Sep 17 00:00:00 2001 From: Yevhenii Date: Fri, 23 May 2025 18:02:19 +0300 Subject: [PATCH 60/79] Add validation for DefaultEdgeRuleChain in DeviceProfile - refactoring --- .../dao/service/validator/DeviceProfileDataValidator.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java index dfd0ee82bc..ca312608d7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java @@ -189,11 +189,11 @@ public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator Date: Fri, 23 May 2025 18:10:33 +0300 Subject: [PATCH 61/79] Add validation for DefaultEdgeRuleChain in DeviceProfile - reverted changes --- .../dao/service/validator/DeviceProfileDataValidator.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java index ca312608d7..dfd0ee82bc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java @@ -189,11 +189,11 @@ public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator Date: Tue, 27 May 2025 15:38:32 +0300 Subject: [PATCH 62/79] Refactoring and improvements for EDQS --- .../service/edqs/DefaultEdqsService.java | 26 +++--- .../src/main/resources/thingsboard.yml | 2 + .../EdqsEntityQueryControllerTest.java | 69 ++++++++++++++- .../server/common/data/edqs/AttributeKv.java | 6 +- .../server/common/data/edqs/EdqsObject.java | 2 +- .../common/data/edqs/EdqsObjectKey.java | 18 ++++ .../server/common/data/edqs/EdqsState.java | 27 +++--- .../server/common/data/edqs/Entity.java | 4 +- .../server/common/data/edqs/LatestTsKv.java | 7 +- .../common/data/edqs/fields/EntityFields.java | 5 +- .../common/data/relation/EntityRelation.java | 8 +- .../server/edqs/processor/EdqsProcessor.java | 19 ++--- .../edqs/state/KafkaEdqsStateService.java | 25 ++++-- ...sConverter.java => DefaultEdqsMapper.java} | 83 ++++++++++++------- .../server/edqs/util/EdqsMapper.java | 30 +++++++ .../server/edqs/util/VersionsStore.java | 13 ++- common/proto/src/main/proto/queue.proto | 1 - .../server/queue/edqs/EdqsConfig.java | 2 + edqs/src/main/resources/edqs.yml | 2 + .../server/edqs/repo/AbstractEDQTest.java | 11 +-- 20 files changed, 267 insertions(+), 93 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObjectKey.java rename common/edqs/src/main/java/org/thingsboard/server/edqs/util/{EdqsConverter.java => DefaultEdqsMapper.java} (80%) create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsMapper.java diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java index 024191a270..92ae38f7e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java @@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.EdqsEventType; import org.thingsboard.server.common.data.edqs.EdqsObject; import org.thingsboard.server.common.data.edqs.EdqsState; +import org.thingsboard.server.common.data.edqs.EdqsState.EdqsApiMode; import org.thingsboard.server.common.data.edqs.EdqsState.EdqsSyncStatus; import org.thingsboard.server.common.data.edqs.EdqsSyncRequest; import org.thingsboard.server.common.data.edqs.Entity; @@ -55,7 +56,8 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.edqs.processor.EdqsProducer; import org.thingsboard.server.edqs.state.EdqsPartitionService; -import org.thingsboard.server.edqs.util.EdqsConverter; +import org.thingsboard.server.edqs.util.DefaultEdqsMapper; +import org.thingsboard.server.edqs.util.EdqsMapper; import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; @@ -82,7 +84,7 @@ import java.util.concurrent.TimeUnit; public class DefaultEdqsService implements EdqsService { private final EdqsClientQueueFactory queueFactory; - private final EdqsConverter edqsConverter; + private final EdqsMapper edqsMapper; private final EdqsSyncService edqsSyncService; private final EdqsApiService edqsApiService; private final DistributedLockService distributedLockService; @@ -182,7 +184,12 @@ public class DefaultEdqsService implements EdqsService { log.info("Processing system msg {}", msg); try { if (msg.getApiEnabled() != null) { - state.setApiEnabled(msg.getApiEnabled()); + if (msg.getApiEnabled()) { + state.setApiMode(EdqsApiMode.ENABLED); + } else { + state.setApiMode(EdqsApiMode.DISABLED); + } + log.info("New state: {}", state); } if (msg.getEdqsReady() != null) { onEdqsReady(msg.getEdqsReady()); @@ -252,8 +259,8 @@ public class DefaultEdqsService implements EdqsService { if (state.isApiReady()) { if (autoEnableApi) { - if (state.getApiEnabled() == null) { - state.setApiEnabled(true); + if (state.getApiMode() == null || state.getApiMode() == EdqsApiMode.AUTO_DISABLED) { + state.setApiMode(EdqsApiMode.AUTO_ENABLED); log.info("New state: {}. Auto-enabled EDQS API", state); } else { log.info("New state: {}. API mode left as is", state); @@ -263,7 +270,7 @@ public class DefaultEdqsService implements EdqsService { } } else { if (state.isApiEnabled()) { - state.setApiEnabled(false); + state.setApiMode(EdqsApiMode.AUTO_DISABLED); log.info("New state: {}. Disabled EDQS API", state); } else { log.info("New state: {}. API left disabled", state); @@ -284,7 +291,7 @@ public class DefaultEdqsService implements EdqsService { log.trace("[{}][{}] Ignoring update event, type {} not supported", tenantId, entityId, entityType); return; } - onUpdate(tenantId, objectType, edqsConverter.toEntity(entityType, entity)); + onUpdate(tenantId, objectType, DefaultEdqsMapper.toEntity(entityType, entity)); } @Override @@ -311,12 +318,11 @@ public class DefaultEdqsService implements EdqsService { protected void processEvent(TenantId tenantId, ObjectType objectType, EdqsEventType eventType, EdqsObject object) { executor.submit(() -> { try { - String key = object.key(); + String key = object.stringKey(); Long version = object.version(); EdqsEventMsg.Builder eventMsg = EdqsEventMsg.newBuilder() - .setKey(key) .setObjectType(objectType.name()) - .setData(ByteString.copyFrom(edqsConverter.serialize(objectType, object))) + .setData(ByteString.copyFrom(edqsMapper.serialize(object))) .setEventType(eventType.name()); if (version != null) { eventMsg.setVersion(version); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index bebddd3f87..ea60d8dc44 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1782,6 +1782,8 @@ queue: max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:20000}" # Thread pool size for EDQS requests executor request_executor_size: "${TB_EDQS_REQUEST_EXECUTOR_SIZE:50}" + # Time to live for EDQS versions cache in minutes. Must be bigger than the time taken for the sync process. + versions_cache_ttl: "${TB_EDQS_VERSIONS_CACHE_TTL_MINUTES:60}" # Strings longer than this threshold will be compressed string_compression_length_threshold: "${TB_EDQS_STRING_COMPRESSION_LENGTH_THRESHOLD:512}" stats: diff --git a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java index b84a324bf7..1bea463159 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java @@ -15,10 +15,15 @@ */ package org.thingsboard.server.controller; +import org.assertj.core.api.ThrowingConsumer; import org.junit.Before; +import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.test.context.TestPropertySource; +import org.thingsboard.server.common.data.edqs.EdqsState; +import org.thingsboard.server.common.data.edqs.EdqsState.EdqsApiMode; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; @@ -26,9 +31,11 @@ import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.edqs.util.EdqsRocksDb; +import org.thingsboard.server.queue.discovery.DiscoveryService; import java.util.concurrent.TimeUnit; +import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; @DaoSqlTest @@ -39,13 +46,16 @@ import static org.awaitility.Awaitility.await; "queue.edqs.api.supported=true", "queue.edqs.api.auto_enable=true", "queue.edqs.mode=local", - "queue.edqs.readiness_check_interval=1000" + "queue.edqs.readiness_check_interval=500" }) public class EdqsEntityQueryControllerTest extends EntityQueryControllerTest { @Autowired private EdqsService edqsService; + @Autowired + private DiscoveryService discoveryService; + @MockBean // so that we don't do backup for tests private EdqsRocksDb edqsRocksDb; @@ -66,4 +76,61 @@ public class EdqsEntityQueryControllerTest extends EntityQueryControllerTest { result -> result == expectedResult); } + @Test + public void testEdqsState() { + assertThat(edqsService.getState().getApiMode()).isEqualTo(EdqsApiMode.AUTO_ENABLED); + + // notifying EDQS is not ready: API should be auto-disabled + discoveryService.setReady(false); + verifyState(state -> { + assertThat(state.getApiMode()).isEqualTo(EdqsApiMode.AUTO_DISABLED); + assertThat(state.getEdqsReady()).isFalse(); + assertThat(state.isApiEnabled()).isFalse(); + }); + + // manually disabling API + edqsService.processSystemRequest(ToCoreEdqsRequest.builder() + .apiEnabled(false) + .build()); + verifyState(state -> { + assertThat(state.getApiMode()).isEqualTo(EdqsApiMode.DISABLED); + assertThat(state.getEdqsReady()).isFalse(); + assertThat(state.isApiEnabled()).isFalse(); + }); + + // notifying EDQS is ready: API should not be enabled automatically because manually disabled previously + discoveryService.setReady(true); + verifyState(state -> { + assertThat(state.getEdqsReady()).isTrue(); + assertThat(state.getApiMode()).isEqualTo(EdqsApiMode.DISABLED); + assertThat(state.isApiEnabled()).isFalse(); + }); + + // manually enabling API + edqsService.processSystemRequest(ToCoreEdqsRequest.builder() + .apiEnabled(true) + .build()); + verifyState(state -> { + assertThat(state.getApiMode()).isEqualTo(EdqsApiMode.ENABLED); + assertThat(state.getEdqsReady()).isTrue(); + assertThat(state.isApiEnabled()).isTrue(); + }); + + // notifying EDQS is not ready: API should be auto-disabled + discoveryService.setReady(false); + verifyState(state -> { + assertThat(state.getApiMode()).isEqualTo(EdqsApiMode.AUTO_DISABLED); + assertThat(state.getEdqsReady()).isFalse(); + assertThat(state.isApiEnabled()).isFalse(); + }); + + discoveryService.setReady(true); + } + + private void verifyState(ThrowingConsumer assertion) { + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + assertThat(edqsService.getState()).satisfies(assertion); + }); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java index c162365257..3ffd6ad424 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java @@ -25,6 +25,8 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; +import java.util.UUID; + @Data @AllArgsConstructor @NoArgsConstructor @@ -58,7 +60,7 @@ public class AttributeKv implements EdqsObject { } @Override - public String key() { + public String stringKey() { return "a_" + entityId + "_" + scope + "_" + key; } @@ -72,4 +74,6 @@ public class AttributeKv implements EdqsObject { return ObjectType.ATTRIBUTE_KV; } + public record Key(UUID entityId, AttributeScope scope, int key) implements EdqsObjectKey {} + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java index a74c90208a..f4685f896a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java @@ -21,7 +21,7 @@ import org.thingsboard.server.common.data.ObjectType; public interface EdqsObject { @JsonIgnore - String key(); + String stringKey(); @JsonIgnore Long version(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObjectKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObjectKey.java new file mode 100644 index 0000000000..8d48bbd383 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObjectKey.java @@ -0,0 +1,18 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +public interface EdqsObjectKey {} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsState.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsState.java index 52778ffd7e..7c3b328022 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsState.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsState.java @@ -18,6 +18,7 @@ package org.thingsboard.server.common.data.edqs; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Getter; import lombok.NoArgsConstructor; +import lombok.Setter; import org.apache.commons.lang3.BooleanUtils; @Getter @@ -26,9 +27,10 @@ import org.apache.commons.lang3.BooleanUtils; public class EdqsState { private Boolean edqsReady; + @Setter private EdqsSyncStatus syncStatus; - - private Boolean apiEnabled; // null until auto-enabled or set manually + @Setter + private EdqsApiMode apiMode; public boolean setEdqsReady(boolean ready) { boolean changed = BooleanUtils.toBooleanDefaultIfNull(this.edqsReady, false) != ready; @@ -36,22 +38,12 @@ public class EdqsState { return changed; } - public void setSyncStatus(EdqsSyncStatus syncStatus) { - this.syncStatus = syncStatus; - } - - public boolean setApiEnabled(boolean apiEnabled) { - boolean changed = BooleanUtils.toBooleanDefaultIfNull(this.apiEnabled, false) != apiEnabled; - this.apiEnabled = apiEnabled; - return changed; - } - public boolean isApiReady() { return edqsReady && syncStatus == EdqsSyncStatus.FINISHED; } public boolean isApiEnabled() { - return apiEnabled != null && apiEnabled; + return apiMode != null && (apiMode == EdqsApiMode.ENABLED || apiMode == EdqsApiMode.AUTO_ENABLED); } @Override @@ -59,7 +51,7 @@ public class EdqsState { return '[' + "EDQS ready: " + edqsReady + ", sync status: " + syncStatus + - ", API enabled: " + apiEnabled + + ", API mode: " + apiMode + ']'; } @@ -70,4 +62,11 @@ public class EdqsState { FAILED } + public enum EdqsApiMode { + ENABLED, + AUTO_ENABLED, + DISABLED, + AUTO_DISABLED + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java index c22ef147e3..96a5b26aee 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java @@ -51,7 +51,7 @@ public class Entity implements EdqsObject { } @Override - public String key() { + public String stringKey() { return "e_" + fields.getId().toString(); } @@ -65,4 +65,6 @@ public class Entity implements EdqsObject { return ObjectType.fromEntityType(type); } + public record Key(UUID id) implements EdqsObjectKey {} + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java index 8bd69c41a4..12695de423 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java @@ -24,6 +24,8 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import java.util.UUID; + @Data @AllArgsConstructor @NoArgsConstructor @@ -53,7 +55,8 @@ public class LatestTsKv implements EdqsObject { this.version = version != null ? version : 0L; } - public String key() { + @Override + public String stringKey() { return "l_" + entityId + "_" + key; } @@ -67,4 +70,6 @@ public class LatestTsKv implements EdqsObject { return ObjectType.LATEST_TS_KV; } + public record Key(UUID entityId, int key) implements EdqsObjectKey {} + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java index 1b0975542c..2eae806f43 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java @@ -29,9 +29,7 @@ public interface EntityFields { Logger log = LoggerFactory.getLogger(EntityFields.class); - default UUID getId() { - return null; - } + UUID getId(); default UUID getTenantId() { return null; @@ -147,6 +145,7 @@ public interface EntityFields { default String getAsString(String key) { return switch (key) { + case "id" -> getId().toString(); case "createdTime" -> Long.toString(getCreatedTime()); case "title" -> getName(); case "type" -> getType(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java b/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java index 8980d0e634..017c9e5010 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java @@ -27,10 +27,12 @@ import org.thingsboard.server.common.data.BaseDataWithAdditionalInfo; import org.thingsboard.server.common.data.HasVersion; import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.EdqsObjectKey; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.validation.Length; import java.io.Serializable; +import java.util.UUID; @Slf4j @Schema @@ -118,8 +120,8 @@ public class EntityRelation implements HasVersion, Serializable, EdqsObject { BaseDataWithAdditionalInfo.setJson(addInfo, json -> this.additionalInfo = json, bytes -> this.additionalInfoBytes = bytes); } - @JsonIgnore - public String key() { + @Override + public String stringKey() { return "r_" + from + "_" + to + "_" + typeGroup + "_" + type; } @@ -133,4 +135,6 @@ public class EntityRelation implements HasVersion, Serializable, EdqsObject { return ObjectType.RELATION; } + public record Key(UUID from, UUID to, RelationTypeGroup typeGroup, String type) implements EdqsObjectKey {} + } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java index 7b30fbaaa5..12256f56ca 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java @@ -45,7 +45,7 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.edqs.repo.EdqsRepository; import org.thingsboard.server.edqs.state.EdqsPartitionService; import org.thingsboard.server.edqs.state.EdqsStateService; -import org.thingsboard.server.edqs.util.EdqsConverter; +import org.thingsboard.server.edqs.util.EdqsMapper; import org.thingsboard.server.edqs.util.VersionsStore; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; @@ -81,7 +81,7 @@ import static org.thingsboard.server.common.msg.queue.TopicPartitionInfo.withTop public class EdqsProcessor implements TbQueueHandler, TbProtoQueueMsg> { private final EdqsQueueFactory queueFactory; - private final EdqsConverter converter; + private final EdqsMapper mapper; private final EdqsRepository repository; private final EdqsConfig config; private final EdqsExecutors edqsExecutors; @@ -92,8 +92,7 @@ public class EdqsProcessor implements TbQueueHandler, private PartitionedQueueConsumerManager> eventConsumer; private PartitionedQueueResponseTemplate, TbProtoQueueMsg> responseTemplate; private ListeningExecutorService requestExecutor; - - private final VersionsStore versionsStore = new VersionsStore(); + private VersionsStore versionsStore; private final AtomicInteger counter = new AtomicInteger(); @Getter @@ -110,6 +109,7 @@ public class EdqsProcessor implements TbQueueHandler, } }; requestExecutor = edqsExecutors.getRequestExecutor(); + versionsStore = new VersionsStore(config.getVersionsCacheTtl()); eventConsumer = PartitionedQueueConsumerManager.>create() .queueKey(new QueueKey(ServiceType.EDQS, config.getEventsTopic())) @@ -224,22 +224,20 @@ public class EdqsProcessor implements TbQueueHandler, TenantId tenantId = getTenantId(edqsMsg); ObjectType objectType = ObjectType.valueOf(eventMsg.getObjectType()); EdqsEventType eventType = EdqsEventType.valueOf(eventMsg.getEventType()); - String key = eventMsg.getKey(); Long version = eventMsg.hasVersion() ? eventMsg.getVersion() : null; + EdqsObject object = mapper.deserialize(objectType, eventMsg.getData().toByteArray(), false); if (version != null) { - if (!versionsStore.isNew(key, version)) { + if (!versionsStore.isNew(mapper.getKey(object), version)) { return; } } else if (!ObjectType.unversionedTypes.contains(objectType)) { - log.warn("[{}] {} {} doesn't have version", tenantId, objectType, key); + log.warn("[{}] {} doesn't have version: {}", tenantId, objectType, object); } if (backup) { - stateService.save(tenantId, objectType, key, eventType, edqsMsg); + stateService.save(tenantId, objectType, object.stringKey(), eventType, edqsMsg); } - EdqsObject object = converter.deserialize(objectType, eventMsg.getData().toByteArray()); - log.debug("[{}] Processing event [{}] [{}] [{}] [{}]", tenantId, objectType, eventType, key, version); int count = counter.incrementAndGet(); if (count % 100000 == 0) { log.info("Processed {} events", count); @@ -251,6 +249,7 @@ public class EdqsProcessor implements TbQueueHandler, .eventType(eventType) .object(object) .build(); + log.debug("Processing event: {}", event); repository.processEvent(event); } } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java index 7ad0258cbb..b506c10473 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java @@ -22,11 +22,13 @@ import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.edqs.EdqsObject; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.edqs.processor.EdqsProcessor; import org.thingsboard.server.edqs.processor.EdqsProducer; +import org.thingsboard.server.edqs.util.EdqsMapper; import org.thingsboard.server.edqs.util.VersionsStore; import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; @@ -63,6 +65,7 @@ public class KafkaEdqsStateService implements EdqsStateService { private final KafkaEdqsQueueFactory queueFactory; private final DiscoveryService discoveryService; private final EdqsExecutors edqsExecutors; + private final EdqsMapper mapper; @Autowired @Lazy private EdqsProcessor edqsProcessor; @@ -71,8 +74,8 @@ public class KafkaEdqsStateService implements EdqsStateService { private KafkaQueueStateService, TbProtoQueueMsg> queueStateService; private QueueConsumerManager> eventsToBackupConsumer; private EdqsProducer stateProducer; + private VersionsStore versionsStore; - private final VersionsStore versionsStore = new VersionsStore(); private final AtomicInteger stateReadCount = new AtomicInteger(); private final AtomicInteger eventsReadCount = new AtomicInteger(); @@ -80,6 +83,7 @@ public class KafkaEdqsStateService implements EdqsStateService { @Override public void init(PartitionedQueueConsumerManager> eventConsumer, List> otherConsumers) { + versionsStore = new VersionsStore(config.getVersionsCacheTtl()); TbKafkaAdmin queueAdmin = queueFactory.getEdqsQueueAdmin(); stateConsumer = PartitionedQueueConsumerManager.>create() .queueKey(new QueueKey(ServiceType.EDQS, config.getStateTopic())) @@ -122,22 +126,25 @@ public class KafkaEdqsStateService implements EdqsStateService { if (msg.hasEventMsg()) { EdqsEventMsg eventMsg = msg.getEventMsg(); - String key = eventMsg.getKey(); - int count = eventsReadCount.incrementAndGet(); - if (count % 100000 == 0) { - log.info("[events-to-backup] Processed {} msgs", count); - } + ObjectType objectType = ObjectType.valueOf(eventMsg.getObjectType()); + EdqsObject object = mapper.deserialize(objectType, eventMsg.getData().toByteArray(), true); + if (eventMsg.hasVersion()) { - if (!versionsStore.isNew(key, eventMsg.getVersion())) { + if (!versionsStore.isNew(mapper.getKey(object), eventMsg.getVersion())) { continue; } } TenantId tenantId = getTenantId(msg); - ObjectType objectType = ObjectType.valueOf(eventMsg.getObjectType()); EdqsEventType eventType = EdqsEventType.valueOf(eventMsg.getEventType()); + String key = object.stringKey(); log.trace("[{}] Saving to backup [{}] [{}] [{}]", tenantId, objectType, eventType, key); - stateProducer.send(tenantId, objectType, key, msg); + stateProducer.send(tenantId, objectType, object.stringKey(), msg); + + int count = eventsReadCount.incrementAndGet(); + if (count % 100000 == 0) { + log.info("[events-to-backup] Processed {} msgs", count); + } } } catch (Throwable t) { log.error("Failed to process message: {}", queueMsg, t); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/DefaultEdqsMapper.java similarity index 80% rename from common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java rename to common/edqs/src/main/java/org/thingsboard/server/edqs/util/DefaultEdqsMapper.java index 9a84436f36..0d3fc35b08 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/DefaultEdqsMapper.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.deser.std.StdDeserializer; -import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.protobuf.ByteString; import lombok.RequiredArgsConstructor; @@ -36,6 +35,7 @@ import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.AttributeKv; import org.thingsboard.server.common.data.edqs.DataPoint; import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.EdqsObjectKey; import org.thingsboard.server.common.data.edqs.Entity; import org.thingsboard.server.common.data.edqs.LatestTsKv; import org.thingsboard.server.common.data.edqs.fields.FieldsUtil; @@ -52,6 +52,7 @@ import org.thingsboard.server.edqs.data.dp.DoubleDataPoint; import org.thingsboard.server.edqs.data.dp.JsonDataPoint; import org.thingsboard.server.edqs.data.dp.LongDataPoint; import org.thingsboard.server.edqs.data.dp.StringDataPoint; +import org.thingsboard.server.edqs.repo.KeyDictionary; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DataPointProto; import org.xerial.snappy.Snappy; @@ -65,19 +66,29 @@ import java.util.UUID; @Service @RequiredArgsConstructor @Slf4j -public class EdqsConverter { +public class DefaultEdqsMapper implements EdqsMapper { private final EdqsStatsService edqsStatsService; @Value("${queue.edqs.string_compression_length_threshold:512}") private int stringCompressionLengthThreshold; - private final Map> converters = new HashMap<>(); - private final Converter defaultConverter = new JsonConverter<>(Entity.class); + private final Map> mappers = new HashMap<>(); + private final Mapper defaultMapper = new JsonMapper<>(Entity.class) { + @Override + public EdqsObjectKey getKey(Entity entity) { + return new Entity.Key(entity.getFields().getId()); + } + }; { - converters.put(ObjectType.RELATION, new JsonConverter<>(EntityRelation.class)); - converters.put(ObjectType.ATTRIBUTE_KV, new Converter() { + mappers.put(ObjectType.RELATION, new JsonMapper<>(EntityRelation.class) { + @Override + public EdqsObjectKey getKey(EntityRelation relation) { + return new EntityRelation.Key(relation.getFrom().getId(), relation.getTo().getId(), relation.getTypeGroup(), relation.getType()); + } + }); + mappers.put(ObjectType.ATTRIBUTE_KV, new Mapper() { @Override public byte[] serialize(ObjectType type, AttributeKv attributeKv) { var proto = TransportProtos.AttributeKvProto.newBuilder() @@ -94,12 +105,12 @@ public class EdqsConverter { } @Override - public AttributeKv deserialize(ObjectType type, byte[] bytes) throws Exception { + public AttributeKv deserialize(ObjectType type, byte[] bytes, boolean onlyKey) throws Exception { TransportProtos.AttributeKvProto proto = TransportProtos.AttributeKvProto.parseFrom(bytes); EntityId entityId = EntityIdFactory.getByTypeAndUuid(ProtoUtils.fromProto(proto.getEntityType()), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); AttributeScope scope = AttributeScope.values()[proto.getScope().getNumber()]; - DataPoint dataPoint = proto.hasDataPoint() ? fromDataPointProto(proto.getDataPoint()) : null; + DataPoint dataPoint = onlyKey || !proto.hasDataPoint() ? null : fromDataPointProto(proto.getDataPoint()); return AttributeKv.builder() .entityId(entityId) .scope(scope) @@ -108,8 +119,13 @@ public class EdqsConverter { .dataPoint(dataPoint) .build(); } + + @Override + public EdqsObjectKey getKey(AttributeKv attributeKv) { + return new AttributeKv.Key(attributeKv.getEntityId().getId(), attributeKv.getScope(), KeyDictionary.get(attributeKv.getKey())); + } }); - converters.put(ObjectType.LATEST_TS_KV, new Converter() { + mappers.put(ObjectType.LATEST_TS_KV, new Mapper() { @Override public byte[] serialize(ObjectType type, LatestTsKv latestTsKv) { var proto = TransportProtos.LatestTsKvProto.newBuilder() @@ -125,11 +141,11 @@ public class EdqsConverter { } @Override - public LatestTsKv deserialize(ObjectType type, byte[] bytes) throws Exception { + public LatestTsKv deserialize(ObjectType type, byte[] bytes, boolean onlyKey) throws Exception { TransportProtos.LatestTsKvProto proto = TransportProtos.LatestTsKvProto.parseFrom(bytes); EntityId entityId = EntityIdFactory.getByTypeAndUuid(ProtoUtils.fromProto(proto.getEntityType()), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - DataPoint dataPoint = proto.hasDataPoint() ? fromDataPointProto(proto.getDataPoint()) : null; + DataPoint dataPoint = onlyKey || !proto.hasDataPoint() ? null : fromDataPointProto(proto.getDataPoint()); return LatestTsKv.builder() .entityId(entityId) .key(proto.getKey()) @@ -137,6 +153,11 @@ public class EdqsConverter { .dataPoint(dataPoint) .build(); } + + @Override + public EdqsObjectKey getKey(LatestTsKv latestTsKv) { + return new LatestTsKv.Key(latestTsKv.getEntityId().getId(), KeyDictionary.get(latestTsKv.getKey())); + } }); } @@ -223,30 +244,30 @@ public class EdqsConverter { @SuppressWarnings("unchecked") @SneakyThrows - public byte[] serialize(ObjectType type, T value) { - Converter converter = (Converter) converters.get(type); - if (converter != null) { - return converter.serialize(type, value); - } else { - return defaultConverter.serialize(type, (Entity) value); - } + public byte[] serialize(T value) { + ObjectType type = value.type(); + Mapper mapper = (Mapper) mappers.getOrDefault(type, defaultMapper); + return mapper.serialize(type, value); } @SneakyThrows - public EdqsObject deserialize(ObjectType type, byte[] bytes) { - Converter converter = converters.get(type); - if (converter != null) { - return converter.deserialize(type, bytes); - } else { - return defaultConverter.deserialize(type, bytes); - } + public EdqsObject deserialize(ObjectType type, byte[] bytes, boolean onlyKey) { + Mapper mapper = mappers.getOrDefault(type, defaultMapper); + return mapper.deserialize(type, bytes, onlyKey); + } + + @SuppressWarnings("unchecked") + @SneakyThrows + public EdqsObjectKey getKey(T object) { + Mapper mapper = (Mapper) mappers.getOrDefault(object.type(), defaultMapper); + return mapper.getKey(object); } @RequiredArgsConstructor - private static class JsonConverter implements Converter { + private static abstract class JsonMapper implements Mapper { private static final SimpleModule module = new SimpleModule(); - private static final ObjectMapper mapper = JsonMapper.builder() + private static final ObjectMapper mapper = com.fasterxml.jackson.databind.json.JsonMapper.builder() .visibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY) .visibility(PropertyAccessor.GETTER, JsonAutoDetect.Visibility.NONE) .visibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE) @@ -267,17 +288,19 @@ public class EdqsConverter { @SneakyThrows @Override - public T deserialize(ObjectType objectType, byte[] bytes) { + public T deserialize(ObjectType objectType, byte[] bytes, boolean onlyKey) { return mapper.readValue(bytes, this.type); } } - private interface Converter { + private interface Mapper { byte[] serialize(ObjectType type, T value) throws Exception; - T deserialize(ObjectType type, byte[] bytes) throws Exception; + T deserialize(ObjectType type, byte[] bytes, boolean onlyKey) throws Exception; + + EdqsObjectKey getKey(T object); } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsMapper.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsMapper.java new file mode 100644 index 0000000000..bd6ed22cd1 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsMapper.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.EdqsObjectKey; + +public interface EdqsMapper { + + byte[] serialize(T value); + + EdqsObject deserialize(ObjectType type, byte[] bytes, boolean onlyKey); + + EdqsObjectKey getKey(T object); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java index 4a09a76e9a..ba3263eec2 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java @@ -18,6 +18,7 @@ package org.thingsboard.server.edqs.util; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.edqs.EdqsObjectKey; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -25,11 +26,15 @@ import java.util.concurrent.atomic.AtomicBoolean; @Slf4j public class VersionsStore { - private final Cache versions = Caffeine.newBuilder() - .expireAfterWrite(24, TimeUnit.HOURS) - .build(); + private final Cache versions; - public boolean isNew(String key, Long version) { + public VersionsStore(int ttlMinutes) { + this.versions = Caffeine.newBuilder() + .expireAfterWrite(ttlMinutes, TimeUnit.MINUTES) + .build(); + } + + public boolean isNew(EdqsObjectKey key, Long version) { AtomicBoolean isNew = new AtomicBoolean(false); versions.asMap().compute(key, (k, prevVersion) -> { if (prevVersion == null || prevVersion <= version) { diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index e874c81444..cc5fc88f72 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1833,7 +1833,6 @@ message FromEdqsMsg { } message EdqsEventMsg { - string key = 1; string objectType = 2; bytes data = 3; string eventType = 4; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java index 4a9b9f2a28..18f46bd358 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java @@ -46,6 +46,8 @@ public class EdqsConfig { private int maxRequestTimeout; @Value("${queue.edqs.request_executor_size:50}") private int requestExecutorSize; + @Value("${queue.edqs.versions_cache_ttl:60}") + private int versionsCacheTtl; public String getLabel() { if (partitioningStrategy == EdqsPartitioningStrategy.NONE) { diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml index 0815187704..9ac8455abb 100644 --- a/edqs/src/main/resources/edqs.yml +++ b/edqs/src/main/resources/edqs.yml @@ -73,6 +73,8 @@ queue: max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:20000}" # Thread pool size for EDQS requests executor request_executor_size: "${TB_EDQS_REQUEST_EXECUTOR_SIZE:50}" + # Time to live for EDQS versions cache in minutes. Must be bigger than the time taken for the sync process. + versions_cache_ttl: "${TB_EDQS_VERSIONS_CACHE_TTL_MINUTES:60}" # Strings longer than this threshold will be compressed string_compression_length_threshold: "${TB_EDQS_STRING_COMPRESSION_LENGTH_THRESHOLD:512}" stats: diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java index 330d22a3c6..a7f80fdffc 100644 --- a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java @@ -60,7 +60,8 @@ import org.thingsboard.server.common.data.query.StringFilterPredicate; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.stats.DummyEdqsStatsService; -import org.thingsboard.server.edqs.util.EdqsConverter; +import org.thingsboard.server.edqs.util.DefaultEdqsMapper; +import org.thingsboard.server.edqs.util.EdqsMapper; import java.util.Collections; import java.util.List; @@ -79,7 +80,7 @@ public abstract class AbstractEDQTest { @Autowired protected DefaultEdqsRepository repository; @Autowired - protected EdqsConverter edqsConverter; + protected EdqsMapper edqsMapper; @MockBean private DummyEdqsStatsService edqsStatsService; @@ -244,12 +245,12 @@ public abstract class AbstractEDQTest { } protected void addOrUpdate(EntityType entityType, Object entity) { - addOrUpdate(EdqsConverter.toEntity(entityType, entity)); + addOrUpdate(DefaultEdqsMapper.toEntity(entityType, entity)); } protected void addOrUpdate(EdqsObject edqsObject) { - byte[] serialized = edqsConverter.serialize(edqsObject.type(), edqsObject); - edqsObject = edqsConverter.deserialize(edqsObject.type(), serialized); + byte[] serialized = edqsMapper.serialize(edqsObject); + edqsObject = edqsMapper.deserialize(edqsObject.type(), serialized, false); repository.get(tenantId).addOrUpdate(edqsObject); } From eaab758c200a2d2a3586fb5b7dc1bcef60f6d821 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 27 May 2025 15:47:28 +0300 Subject: [PATCH 63/79] Fix EDQS state topic prefix usage --- .../org/thingsboard/server/edqs/processor/EdqsProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java index cfffe21ff9..133a2f171b 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java @@ -150,7 +150,7 @@ public class EdqsProcessor implements TbQueueHandler, } try { Set newPartitions = event.getNewPartitions().get(new QueueKey(ServiceType.EDQS)); - stateService.process(withTopic(newPartitions, config.getStateTopic())); + stateService.process(withTopic(newPartitions, topicService.buildTopicName(config.getStateTopic()))); // partitions for event and request consumers are updated by stateService Set oldPartitions = event.getOldPartitions().get(new QueueKey(ServiceType.EDQS)); From acae24e9603180ac0c935fa99178868503b71750 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 27 May 2025 18:05:37 +0300 Subject: [PATCH 64/79] Fix EntityNameQueryProcessor --- .../org/thingsboard/server/edqs/processor/EdqsProcessor.java | 3 +++ .../server/edqs/query/processor/EntityNameQueryProcessor.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java index 133a2f171b..9fe6b2843e 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java @@ -55,6 +55,7 @@ import org.thingsboard.server.queue.TbQueueHandler; import org.thingsboard.server.queue.common.PartitionedQueueResponseTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; +import org.thingsboard.server.queue.discovery.DiscoveryService; import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; @@ -87,6 +88,7 @@ public class EdqsProcessor implements TbQueueHandler, private final EdqsConfig config; private final EdqsExecutors edqsExecutors; private final EdqsPartitionService partitionService; + private final DiscoveryService discoveryService; private final TopicService topicService; private final ConfigurableApplicationContext applicationContext; private final EdqsStateService stateService; @@ -106,6 +108,7 @@ public class EdqsProcessor implements TbQueueHandler, if (error instanceof OutOfMemoryError) { log.error("OOM detected, shutting down"); repository.clear(); + discoveryService.setReady(false); Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("edqs-shutdown")) .execute(applicationContext::close); } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java index ec88db4d0f..fbb36cf2a0 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java @@ -35,7 +35,7 @@ public class EntityNameQueryProcessor extends AbstractSimpleQueryProcessor Date: Wed, 28 May 2025 20:24:48 +0300 Subject: [PATCH 65/79] Refactoring for TelemetrySubscriptionUpdate --- .../TbAbstractEntityQuerySubCtx.java | 12 +++-- .../subscription/TbAlarmDataSubCtx.java | 6 +-- .../subscription/TbEntityDataSubCtx.java | 40 +++++++-------- .../service/ws/DefaultWebSocketService.java | 2 +- .../sub/TelemetrySubscriptionUpdate.java | 50 ++++++++++++------- 5 files changed, 61 insertions(+), 49 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAbstractEntityQuerySubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAbstractEntityQuerySubCtx.java index 39e486e0a8..2c3127bc4d 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAbstractEntityQuerySubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAbstractEntityQuerySubCtx.java @@ -44,6 +44,7 @@ import org.thingsboard.server.service.ws.WebSocketSessionRef; import org.thingsboard.server.service.ws.telemetry.sub.TelemetrySubscriptionUpdate; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -154,9 +155,8 @@ public abstract class TbAbstractEntityQuerySubCtx ex private void dynamicValueSubUpdate(String sessionId, TelemetrySubscriptionUpdate subscriptionUpdate, Map dynamicValueKeySubMap) { Map latestUpdate = new HashMap<>(); - subscriptionUpdate.getData().forEach((k, v) -> { - Object[] data = (Object[]) v.get(0); - latestUpdate.put(k, new TsValue((Long) data[0], (String) data[1])); + subscriptionUpdate.getData().forEach((key, values) -> { + latestUpdate.put(key, getLatest(values)); }); boolean invalidateFilter = false; @@ -283,6 +283,12 @@ public abstract class TbAbstractEntityQuerySubCtx ex } } + protected TsValue getLatest(List values) { + return values.stream() + .max(Comparator.comparing(TsValue::getTs)) + .orElse(null); + } + @Data public static class DynamicValueKey { @Getter diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java index f6b4067543..b9956288a4 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java @@ -38,7 +38,6 @@ import org.thingsboard.server.common.data.query.TsValue; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; -import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.sql.query.EntityKeyMapping; import org.thingsboard.server.service.ws.WebSocketService; import org.thingsboard.server.service.ws.WebSocketSessionRef; @@ -191,9 +190,8 @@ public class TbAlarmDataSubCtx extends TbAbstractDataSubCtx { EntityId entityId = subToEntityIdMap.get(subscriptionUpdate.getSubscriptionId()); if (entityId != null) { Map latestUpdate = new HashMap<>(); - subscriptionUpdate.getData().forEach((k, v) -> { - Object[] data = (Object[]) v.get(0); - latestUpdate.put(k, new TsValue((Long) data[0], (String) data[1])); + subscriptionUpdate.getData().forEach((key, values) -> { + latestUpdate.put(key, getLatest(values)); }); EntityData entityData = entitiesMap.get(entityId); entityData.getLatest().computeIfAbsent(keyType, tmp -> new HashMap<>()).putAll(latestUpdate); diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbEntityDataSubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbEntityDataSubCtx.java index 883b5307d2..6e3ebdc13b 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbEntityDataSubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbEntityDataSubCtx.java @@ -43,7 +43,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -94,16 +93,12 @@ public class TbEntityDataSubCtx extends TbAbstractDataSubCtx { private void sendLatestWsMsg(EntityId entityId, String sessionId, TelemetrySubscriptionUpdate subscriptionUpdate, EntityKeyType keyType) { Map latestUpdate = new HashMap<>(); - subscriptionUpdate.getData().forEach((key, data) -> data.stream() - .filter(o -> o instanceof Object[] && ((Object[]) o).length > 0 && ((Object[]) o)[0] instanceof Long) - .max(Comparator.comparingLong(o -> (Long) ((Object[]) o)[0])) - .ifPresent(max -> { - Object[] arr = (Object[]) max; - latestUpdate.put(key, new TsValue((Long) arr[0], (String) arr[1])); - })); + subscriptionUpdate.getData().forEach((key, values) -> { + latestUpdate.put(key, getLatest(values)); + }); EntityData entityData = getDataForEntity(entityId); if (entityData != null && entityData.getLatest() != null) { - Map latestCtxValues = entityData.getLatest().computeIfAbsent(keyType, key -> new HashMap<>()); + Map latestCtxValues = entityData.getLatest().computeIfAbsent(keyType, __ -> new HashMap<>()); log.trace("[{}][{}][{}] Going to compare update with {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), latestCtxValues); latestCtxValues.forEach((k, v) -> { TsValue update = latestUpdate.get(k); @@ -134,40 +129,39 @@ public class TbEntityDataSubCtx extends TbAbstractDataSubCtx { private void sendTsWsMsg(EntityId entityId, String sessionId, TelemetrySubscriptionUpdate subscriptionUpdate, EntityKeyType keyType) { Map> tsUpdate = new HashMap<>(); - subscriptionUpdate.getData().forEach((k, v) -> { - Object[] data = (Object[]) v.get(0); - tsUpdate.computeIfAbsent(k, key -> new ArrayList<>()).add(new TsValue((Long) data[0], (String) data[1])); + subscriptionUpdate.getData().forEach((key, values) -> { + tsUpdate.put(key, new ArrayList<>(values)); }); Map latestCtxValues = getLatestTsValuesForEntity(entityId); log.trace("[{}][{}][{}] Going to compare update with {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), latestCtxValues); if (latestCtxValues != null) { - latestCtxValues.forEach((k, v) -> { - List updateList = tsUpdate.get(k); + latestCtxValues.forEach((key, latest) -> { + List updateList = tsUpdate.get(key); if (updateList != null) { for (TsValue update : new ArrayList<>(updateList)) { - if (update.getTs() < v.getTs()) { - log.trace("[{}][{}][{}] Removed stale update for key: {} and ts: {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), k, update.getTs()); + if (update.getTs() < latest.getTs()) { + log.trace("[{}][{}][{}] Removed stale update for key: {} and ts: {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), key, update.getTs()); // Looks like this is redundant feature and our UI is ready to merge the updates. //updateList.remove(update); - } else if ((update.getTs() == v.getTs() && update.getValue().equals(v.getValue()))) { - log.trace("[{}][{}][{}] Removed duplicate update for key: {} and ts: {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), k, update.getTs()); + } else if ((update.getTs() == latest.getTs() && update.getValue().equals(latest.getValue()))) { + log.trace("[{}][{}][{}] Removed duplicate update for key: {} and ts: {}", sessionId, cmdId, subscriptionUpdate.getSubscriptionId(), key, update.getTs()); updateList.remove(update); } if (updateList.isEmpty()) { - tsUpdate.remove(k); + tsUpdate.remove(key); } } } }); //Setting new values - tsUpdate.forEach((k, v) -> { - Optional maxValue = v.stream().max(Comparator.comparingLong(TsValue::getTs)); - maxValue.ifPresent(max -> latestCtxValues.put(k, max)); + tsUpdate.forEach((key, values) -> { + values.stream().max(Comparator.comparingLong(TsValue::getTs)) + .ifPresent(latest -> latestCtxValues.put(key, latest)); }); } if (!tsUpdate.isEmpty()) { Map tsMap = new HashMap<>(); - tsUpdate.forEach((key, tsValue) -> tsMap.put(key, tsValue.toArray(new TsValue[tsValue.size()]))); + tsUpdate.forEach((key, values) -> tsMap.put(key, values.toArray(new TsValue[0]))); EntityData entityData = new EntityData(entityId, null, tsMap); sendWsMsg(new EntityDataUpdate(cmdId, null, Collections.singletonList(entityData), maxEntitiesPerDataSubscription)); } diff --git a/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java index 2e4fe9730a..cbe6663663 100644 --- a/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java @@ -274,7 +274,7 @@ public class DefaultWebSocketService implements WebSocketService { @Override public void sendUpdate(String sessionId, int cmdId, TelemetrySubscriptionUpdate update) { // We substitute the subscriptionId with cmdId for old-style subscriptions. - doSendUpdate(sessionId, cmdId, update.copyWithNewSubscriptionId(cmdId)); + doSendUpdate(sessionId, cmdId, update.withSubscriptionId(cmdId)); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/ws/telemetry/sub/TelemetrySubscriptionUpdate.java b/application/src/main/java/org/thingsboard/server/service/ws/telemetry/sub/TelemetrySubscriptionUpdate.java index b22b021a03..6fca1a159a 100644 --- a/application/src/main/java/org/thingsboard/server/service/ws/telemetry/sub/TelemetrySubscriptionUpdate.java +++ b/application/src/main/java/org/thingsboard/server/service/ws/telemetry/sub/TelemetrySubscriptionUpdate.java @@ -16,12 +16,16 @@ package org.thingsboard.server.service.ws.telemetry.sub; import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.With; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.query.TsValue; import org.thingsboard.server.service.subscription.SubscriptionErrorCode; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -29,8 +33,13 @@ import java.util.stream.Collectors; @AllArgsConstructor public class TelemetrySubscriptionUpdate { + + @Getter + @With private final int subscriptionId; + @Getter private int errorCode; + @Getter private String errorMsg; private Map> data; @@ -66,11 +75,27 @@ public class TelemetrySubscriptionUpdate { this.errorMsg = errorMsg != null ? errorMsg : errorCode.getDefaultMsg(); } - public int getSubscriptionId() { - return subscriptionId; - } + public Map> getData() { + if (data == null || data.isEmpty()) { + return Collections.emptyMap(); + } - public Map> getData() { + Map> data = new HashMap<>(); + this.data.forEach((key, entries) -> { + if (entries.isEmpty()) { + return; + } + + List values = new ArrayList<>(entries.size()); + entries.forEach(object -> { + if (!(object instanceof Object[] entry) || entry.length < 2) { + return; + } + TsValue tsValue = new TsValue((Long) entry[0], (String) entry[1]); + values.add(tsValue); + }); + data.put(key, values); + }); return data; } @@ -86,28 +111,17 @@ public class TelemetrySubscriptionUpdate { } } - public int getErrorCode() { - return errorCode; - } - - public String getErrorMsg() { - return errorMsg; - } - - public TelemetrySubscriptionUpdate copyWithNewSubscriptionId(int subscriptionId){ - return new TelemetrySubscriptionUpdate(subscriptionId, errorCode, errorMsg, data); - } - @Override public String toString() { StringBuilder result = new StringBuilder("TelemetrySubscriptionUpdate [subscriptionId=" + subscriptionId + ", errorCode=" + errorCode + ", errorMsg=" + errorMsg + ", data="); data.forEach((k, v) -> { result.append(k).append("=["); - for(Object a : v){ - result.append(Arrays.toString((Object[])a)).append("|"); + for (Object a : v) { + result.append(Arrays.toString((Object[]) a)).append("|"); } result.append("]"); }); return result.toString(); } + } From da6fa2c00b29638eb9906ab3922b561a1c098c61 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 29 May 2025 11:47:11 +0300 Subject: [PATCH 66/79] Refactor TelemetrySubscriptionUpdate --- .../service/subscription/TbAbstractEntityQuerySubCtx.java | 2 +- .../server/service/subscription/TbAlarmDataSubCtx.java | 2 +- .../server/service/subscription/TbEntityDataSubCtx.java | 4 ++-- .../ws/telemetry/sub/TelemetrySubscriptionUpdate.java | 6 ++---- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAbstractEntityQuerySubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAbstractEntityQuerySubCtx.java index 2c3127bc4d..9d51c2ec4a 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAbstractEntityQuerySubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAbstractEntityQuerySubCtx.java @@ -155,7 +155,7 @@ public abstract class TbAbstractEntityQuerySubCtx ex private void dynamicValueSubUpdate(String sessionId, TelemetrySubscriptionUpdate subscriptionUpdate, Map dynamicValueKeySubMap) { Map latestUpdate = new HashMap<>(); - subscriptionUpdate.getData().forEach((key, values) -> { + subscriptionUpdate.getValues().forEach((key, values) -> { latestUpdate.put(key, getLatest(values)); }); diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java index b9956288a4..e896b520bc 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java @@ -190,7 +190,7 @@ public class TbAlarmDataSubCtx extends TbAbstractDataSubCtx { EntityId entityId = subToEntityIdMap.get(subscriptionUpdate.getSubscriptionId()); if (entityId != null) { Map latestUpdate = new HashMap<>(); - subscriptionUpdate.getData().forEach((key, values) -> { + subscriptionUpdate.getValues().forEach((key, values) -> { latestUpdate.put(key, getLatest(values)); }); EntityData entityData = entitiesMap.get(entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbEntityDataSubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbEntityDataSubCtx.java index 6e3ebdc13b..3df03283d7 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbEntityDataSubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbEntityDataSubCtx.java @@ -93,7 +93,7 @@ public class TbEntityDataSubCtx extends TbAbstractDataSubCtx { private void sendLatestWsMsg(EntityId entityId, String sessionId, TelemetrySubscriptionUpdate subscriptionUpdate, EntityKeyType keyType) { Map latestUpdate = new HashMap<>(); - subscriptionUpdate.getData().forEach((key, values) -> { + subscriptionUpdate.getValues().forEach((key, values) -> { latestUpdate.put(key, getLatest(values)); }); EntityData entityData = getDataForEntity(entityId); @@ -129,7 +129,7 @@ public class TbEntityDataSubCtx extends TbAbstractDataSubCtx { private void sendTsWsMsg(EntityId entityId, String sessionId, TelemetrySubscriptionUpdate subscriptionUpdate, EntityKeyType keyType) { Map> tsUpdate = new HashMap<>(); - subscriptionUpdate.getData().forEach((key, values) -> { + subscriptionUpdate.getValues().forEach((key, values) -> { tsUpdate.put(key, new ArrayList<>(values)); }); Map latestCtxValues = getLatestTsValuesForEntity(entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/ws/telemetry/sub/TelemetrySubscriptionUpdate.java b/application/src/main/java/org/thingsboard/server/service/ws/telemetry/sub/TelemetrySubscriptionUpdate.java index 6fca1a159a..e24ab368ec 100644 --- a/application/src/main/java/org/thingsboard/server/service/ws/telemetry/sub/TelemetrySubscriptionUpdate.java +++ b/application/src/main/java/org/thingsboard/server/service/ws/telemetry/sub/TelemetrySubscriptionUpdate.java @@ -31,15 +31,13 @@ import java.util.Map; import java.util.TreeMap; import java.util.stream.Collectors; +@Getter @AllArgsConstructor public class TelemetrySubscriptionUpdate { - @Getter @With private final int subscriptionId; - @Getter private int errorCode; - @Getter private String errorMsg; private Map> data; @@ -75,7 +73,7 @@ public class TelemetrySubscriptionUpdate { this.errorMsg = errorMsg != null ? errorMsg : errorCode.getDefaultMsg(); } - public Map> getData() { + public Map> getValues() { if (data == null || data.isEmpty()) { return Collections.emptyMap(); } From 752e543c4170b52f6242c106baa008d5e291436c Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 30 May 2025 11:25:42 +0300 Subject: [PATCH 67/79] Add JsonIgnore to TelemetrySubscriptionUpdate.getValues --- .../service/ws/telemetry/sub/TelemetrySubscriptionUpdate.java | 2 ++ .../src/test/java/org/thingsboard/server/msa/WsClient.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/ws/telemetry/sub/TelemetrySubscriptionUpdate.java b/application/src/main/java/org/thingsboard/server/service/ws/telemetry/sub/TelemetrySubscriptionUpdate.java index e24ab368ec..4959c5a348 100644 --- a/application/src/main/java/org/thingsboard/server/service/ws/telemetry/sub/TelemetrySubscriptionUpdate.java +++ b/application/src/main/java/org/thingsboard/server/service/ws/telemetry/sub/TelemetrySubscriptionUpdate.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.ws.telemetry.sub; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.With; @@ -73,6 +74,7 @@ public class TelemetrySubscriptionUpdate { this.errorMsg = errorMsg != null ? errorMsg : errorCode.getDefaultMsg(); } + @JsonIgnore public Map> getValues() { if (data == null || data.isEmpty()) { return Collections.emptyMap(); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/WsClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/WsClient.java index 748335cd06..25b935a2fe 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/WsClient.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/WsClient.java @@ -61,7 +61,7 @@ public class WsClient extends WebSocketClient { latch.countDown(); } } catch (IOException e) { - log.error("ws message can't be read"); + log.error("ws message can't be read", e); } } } From 8bb9dd7954996a38ef3ecfbae62153e6e40c8d8f Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 30 May 2025 11:44:00 +0300 Subject: [PATCH 68/79] Fix EntityFields.getCustomerId --- .../common/data/edqs/fields/AbstractEntityFields.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AbstractEntityFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AbstractEntityFields.java index 4aad5eb4dd..cb1aa9300d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AbstractEntityFields.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AbstractEntityFields.java @@ -17,7 +17,7 @@ package org.thingsboard.server.common.data.edqs.fields; import lombok.Data; import lombok.experimental.SuperBuilder; -import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; import java.util.UUID; @@ -36,7 +36,7 @@ public class AbstractEntityFields implements EntityFields { this.id = id; this.createdTime = createdTime; this.tenantId = tenantId; - this.customerId = (customerId != null && customerId != CustomerId.NULL_UUID) ? customerId : null; + this.customerId = checkId(customerId); this.name = name; this.version = version; } @@ -62,4 +62,8 @@ public class AbstractEntityFields implements EntityFields { this(id, createdTime, tenantId, null, null, null); } + protected UUID checkId(UUID id) { + return id != null && !id.equals(EntityId.NULL_UUID) ? id : null; + } + } From 9b6bfd116a491084ed7d78eaf0d4e4e93f24f3da Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 23 May 2025 11:37:42 +0200 Subject: [PATCH 69/79] remove FILLFACTOR 95. As default 90 is fine for most applications --- .../resources/sql/schema-entities-idx-psql-addon.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql b/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql index b756d282f8..ae70e8c725 100644 --- a/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql +++ b/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql @@ -22,19 +22,19 @@ -- That why we need to define DESC index explicitly as (ts DESC NULLS LAST) CREATE INDEX IF NOT EXISTS idx_rule_node_debug_event_main - ON rule_node_debug_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); + ON rule_node_debug_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST); CREATE INDEX IF NOT EXISTS idx_rule_chain_debug_event_main - ON rule_chain_debug_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); + ON rule_chain_debug_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST); CREATE INDEX IF NOT EXISTS idx_stats_event_main - ON stats_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); + ON stats_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST); CREATE INDEX IF NOT EXISTS idx_lc_event_main - ON lc_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); + ON lc_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST); CREATE INDEX IF NOT EXISTS idx_error_event_main - ON error_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); + ON error_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST); CREATE INDEX IF NOT EXISTS idx_cf_debug_event_main - ON cf_debug_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); + ON cf_debug_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST); From 513566bd7b666f9712c555d1ddf5c6f46c92f2c9 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 23 May 2025 11:13:18 +0300 Subject: [PATCH 70/79] Docker file improvements to support installation without mounting the configuration folder (cherry picked from commit a6997cb4a1e61ebae64ab05a63d8dff4b6f0dded) --- msa/tb-node/docker/Dockerfile | 4 ++- msa/tb-node/docker/logback.xml | 38 +++++++++++++++++++++++++++++ msa/tb-node/docker/start-tb-node.sh | 18 ++++++++------ 3 files changed, 52 insertions(+), 8 deletions(-) create mode 100644 msa/tb-node/docker/logback.xml diff --git a/msa/tb-node/docker/Dockerfile b/msa/tb-node/docker/Dockerfile index dfca56acc7..013a37ef9c 100644 --- a/msa/tb-node/docker/Dockerfile +++ b/msa/tb-node/docker/Dockerfile @@ -16,12 +16,14 @@ FROM thingsboard/openjdk17:bookworm-slim -COPY start-tb-node.sh ${pkg.name}.deb /tmp/ +COPY logback.xml start-tb-node.sh ${pkg.name}.deb /tmp/ RUN chmod a+x /tmp/*.sh \ && mv /tmp/start-tb-node.sh /usr/bin && \ (yes | dpkg -i /tmp/${pkg.name}.deb) && \ rm /tmp/${pkg.name}.deb && \ + mv /tmp/logback.xml ${pkg.installFolder}/conf && \ + chown -R ${pkg.user}:${pkg.user} ${pkg.installFolder}/conf/logback.xml && \ (systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || :) && \ chown -R ${pkg.user}:${pkg.user} /tmp && \ chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar diff --git a/msa/tb-node/docker/logback.xml b/msa/tb-node/docker/logback.xml new file mode 100644 index 0000000000..269cb89396 --- /dev/null +++ b/msa/tb-node/docker/logback.xml @@ -0,0 +1,38 @@ + + + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + + + diff --git a/msa/tb-node/docker/start-tb-node.sh b/msa/tb-node/docker/start-tb-node.sh index 7de30564c9..77221398be 100755 --- a/msa/tb-node/docker/start-tb-node.sh +++ b/msa/tb-node/docker/start-tb-node.sh @@ -15,14 +15,21 @@ # limitations under the License. # -CONF_FOLDER="/config" jarfile=${pkg.installFolder}/bin/${pkg.name}.jar configfile=${pkg.name}.conf run_user=${pkg.user} -source "${CONF_FOLDER}/${configfile}" +CONF_FOLDER="/config" +if [ -d "${CONF_FOLDER}" ]; then + LOGGING_CONFIG="${CONF_FOLDER}/logback.xml" + source "${CONF_FOLDER}/${configfile}" + export LOADER_PATH=${CONF_FOLDER},${LOADER_PATH} +else + CONF_FOLDER="/usr/share/${pkg.name}/conf" + LOGGING_CONFIG="/usr/share/${pkg.name}/conf/logback.xml" + source "${CONF_FOLDER}/${configfile}" +fi -export LOADER_PATH=/config,${LOADER_PATH} cd ${pkg.installFolder}/bin @@ -38,7 +45,6 @@ if [ "$INSTALL_TB" == "true" ]; then exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.ThingsboardInstallApplication \ -Dinstall.load_demo=${loadDemo} \ - -Dspring.jpa.hibernate.ddl-auto=none \ -Dinstall.upgrade=false \ -Dlogging.config=/usr/share/thingsboard/bin/install/logback.xml \ org.springframework.boot.loader.launch.PropertiesLauncher @@ -51,7 +57,6 @@ elif [ "$UPGRADE_TB" == "true" ]; then fromVersion="${FROM_VERSION// }" exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.ThingsboardInstallApplication \ - -Dspring.jpa.hibernate.ddl-auto=none \ -Dinstall.upgrade=true \ -Dinstall.upgrade.from_version=${fromVersion} \ -Dlogging.config=/usr/share/thingsboard/bin/install/logback.xml \ @@ -62,8 +67,7 @@ else echo "Starting '${project.name}' ..." exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.ThingsboardServerApplication \ - -Dspring.jpa.hibernate.ddl-auto=none \ - -Dlogging.config=/config/logback.xml \ + -Dlogging.config=${LOGGING_CONFIG} \ org.springframework.boot.loader.launch.PropertiesLauncher fi From 402cf9b5aafd75f11017afcfee263b037feaeeba Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 13 Jun 2025 14:13:01 +0200 Subject: [PATCH 71/79] CVE-2025-49146 postgresql.driver 4.7.5 -> 4.7.7 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0260dc09be..5093c19435 100755 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,7 @@ 1.19.0 1.78.1 2.0.1 - 42.7.5 + 42.7.7 org/thingsboard/server/gen/**/*, org/thingsboard/server/extensions/core/plugin/telemetry/gen/**/* From 6e4ee1eb44a049cf401b51c66bf469bf61f721d0 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 13 Jun 2025 14:46:10 +0200 Subject: [PATCH 72/79] CVE-2025-27817 kafka client 3.7.2 -> 3.9.1 (NetworkReceive.java has no code changes in the Kafka upstream) --- .../java/org/apache/kafka/common/network/NetworkReceive.java | 4 ++-- pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/apache/kafka/common/network/NetworkReceive.java b/application/src/main/java/org/apache/kafka/common/network/NetworkReceive.java index 80192520ca..88a03f5fc8 100644 --- a/application/src/main/java/org/apache/kafka/common/network/NetworkReceive.java +++ b/application/src/main/java/org/apache/kafka/common/network/NetworkReceive.java @@ -103,13 +103,13 @@ public class NetworkReceive implements Receive { if (maxSize != UNLIMITED && receiveSize > maxSize) { throw new ThingsboardKafkaClientError("Invalid receive (size = " + receiveSize + " larger than " + maxSize + ")"); } - requestedBufferSize = receiveSize; //may be 0 for some payloads (SASL) + requestedBufferSize = receiveSize; // may be 0 for some payloads (SASL) if (receiveSize == 0) { buffer = EMPTY_BUFFER; } } } - if (buffer == null && requestedBufferSize != -1) { //we know the size we want but havent been able to allocate it yet + if (buffer == null && requestedBufferSize != -1) { // we know the size we want but haven't been able to allocate it yet if (requestedBufferSize > TB_LOG_REQUESTED_BUFFER_SIZE) { String stackTrace = Arrays.stream(Thread.currentThread().getStackTrace()).map(StackTraceElement::toString).collect(Collectors.joining("|")); log.error("Allocating buffer of size {} for source {}", requestedBufferSize, source); diff --git a/pom.xml b/pom.xml index 5093c19435..66b153d63a 100755 --- a/pom.xml +++ b/pom.xml @@ -113,7 +113,7 @@ - 3.7.2 + 3.9.1 8.10.1 3.5.3 2.2 From 6a5fd0d45c171a06048c39bb5e2f58a038304627 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 13 Jun 2025 15:32:44 +0200 Subject: [PATCH 73/79] CVE-2024-12798 logback 1.5.5 -> 1.5.18 --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 66b153d63a..88efe6723f 100755 --- a/pom.xml +++ b/pom.xml @@ -52,9 +52,9 @@ 6.3.8 5.1.5 0.12.5 - 2.0.13 - 2.23.1 - 1.5.5 + 2.0.17 + 2.24.3 + 1.5.18 0.10 4.17.0 4.2.25 From 1d1a72d54dd0678fca776f555fa0ed6fda0a8aae Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 13 Jun 2025 15:35:06 +0200 Subject: [PATCH 74/79] CVE-2024-51504 org.apache.zookeeper:zookeeper@3.9.3 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 88efe6723f..011b1c967b 100755 --- a/pom.xml +++ b/pom.xml @@ -83,7 +83,7 @@ 2.3.32 2.0.1 5.6.0 - 3.9.2 + 3.9.3 3.25.5 1.63.0 1.2.6 From b69460ab01f86edbe81c5aef5f8220dcc1678942 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 13 Jun 2025 15:42:13 +0200 Subject: [PATCH 75/79] CVE-2025-4949 org.eclipse.jgit:org.eclipse.jgit@6.10.1.202505221210-r --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 011b1c967b..fd21733bd3 100755 --- a/pom.xml +++ b/pom.xml @@ -158,7 +158,7 @@ 2.12.0 3.0.2 - 6.9.0.202403050737-r + 6.10.1.202505221210-r 0.4.8 1.0.0 From 14b3df350260142f856f0e62a1974e92dff981e7 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 13 Jun 2025 15:44:51 +0200 Subject: [PATCH 76/79] CVE-2025-46701 org.apache.tomcat.embed:tomcat-embed-core@10.1.42 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fd21733bd3..94cdff0e9f 100755 --- a/pom.xml +++ b/pom.xml @@ -42,7 +42,7 @@ 4.0.2 2.4.0-b180830.0359 4.0.5 - 10.1.40 + 10.1.42 2.5.2 3.2.12 3.2.12 From 56da48e2075743f212ec4613f07e90fa07eee6ef Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 13 Jun 2025 15:49:47 +0200 Subject: [PATCH 77/79] CVE-2025-41234 org.springframework:spring-web@6.1.15 -> org.springframework:spring-web@6.1.21 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 94cdff0e9f..f38ed581a1 100755 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ 3.2.12 3.2.12 3.2.12 - 6.1.15 + 6.1.21 6.2.11 6.3.8 5.1.5 @@ -1173,7 +1173,7 @@ org.springframework.boot - spring-boot-starter + spring-boot-starter ${spring-boot.version} From a39547ddeedef581aad8cd57870c41b5dd224ee3 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 13 Jun 2025 16:08:11 +0200 Subject: [PATCH 78/79] CVE-2025-22234 org.springframework.security:spring-security-crypto@6.3.9 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f38ed581a1..3e0a67ab0d 100755 --- a/pom.xml +++ b/pom.xml @@ -49,7 +49,7 @@ 3.2.12 6.1.21 6.2.11 - 6.3.8 + 6.3.9 5.1.5 0.12.5 2.0.17 From be789ebd1ffa6f968247df3bb501e1d2bd87dcbc Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 13 Jun 2025 18:37:25 +0200 Subject: [PATCH 79/79] downgrade logback as the least compatible version with Spring Boot 3.2.12 is logback 1.5.6 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3e0a67ab0d..4ddbe5ff0b 100755 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,7 @@ 0.12.5 2.0.17 2.24.3 - 1.5.18 + 1.5.6 0.10 4.17.0 4.2.25