From 44f4b301dd6b70d9ccab385620bed970119fd02a Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Tue, 18 Apr 2023 11:38:40 +0300 Subject: [PATCH 01/10] UI: Fixed color picker input width --- .../components/color-picker/color-picker.component.scss | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ui-ngx/src/app/shared/components/color-picker/color-picker.component.scss b/ui-ngx/src/app/shared/components/color-picker/color-picker.component.scss index f71707195a..ea2bfa94ba 100644 --- a/ui-ngx/src/app/shared/components/color-picker/color-picker.component.scss +++ b/ui-ngx/src/app/shared/components/color-picker/color-picker.component.scss @@ -58,6 +58,10 @@ .color-input { flex: 1; + width: min-content; + &> * { + color: initial; + } } .type-btn { From 448fe341cfe97920463f49ca2b741a1e25718183 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 17 Apr 2023 10:33:22 +0200 Subject: [PATCH 02/10] kafka settings: added default values for values for TbKafkaSettings class --- .../server/queue/kafka/TbKafkaSettings.java | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index 5c63fe6e7d..57be596da7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -53,34 +53,34 @@ public class TbKafkaSettings { @Value("${queue.kafka.ssl.enabled:false}") private boolean sslEnabled; - @Value("${queue.kafka.ssl.truststore.location}") + @Value("${queue.kafka.ssl.truststore.location:}") private String sslTruststoreLocation; - @Value("${queue.kafka.ssl.truststore.password}") + @Value("${queue.kafka.ssl.truststore.password:}") private String sslTruststorePassword; - @Value("${queue.kafka.ssl.keystore.location}") + @Value("${queue.kafka.ssl.keystore.location:}") private String sslKeystoreLocation; - @Value("${queue.kafka.ssl.keystore.password}") + @Value("${queue.kafka.ssl.keystore.password:}") private String sslKeystorePassword; - @Value("${queue.kafka.ssl.key.password}") + @Value("${queue.kafka.ssl.key.password:}") private String sslKeyPassword; - @Value("${queue.kafka.acks}") + @Value("${queue.kafka.acks:all}") private String acks; - @Value("${queue.kafka.retries}") + @Value("${queue.kafka.retries:1}") private int retries; @Value("${queue.kafka.compression.type:none}") private String compressionType; - @Value("${queue.kafka.batch.size}") + @Value("${queue.kafka.batch.size:16384}") private int batchSize; - @Value("${queue.kafka.linger.ms}") + @Value("${queue.kafka.linger.ms:1}") private long lingerMs; @Value("${queue.kafka.max.request.size:1048576}") @@ -89,10 +89,10 @@ public class TbKafkaSettings { @Value("${queue.kafka.max.in.flight.requests.per.connection:5}") private int maxInFlightRequestsPerConnection; - @Value("${queue.kafka.buffer.memory}") + @Value("${queue.kafka.buffer.memory:33554432}") private long bufferMemory; - @Value("${queue.kafka.replication_factor}") + @Value("${queue.kafka.replication_factor:1}") @Getter private short replicationFactor; @@ -111,16 +111,16 @@ public class TbKafkaSettings { @Value("${queue.kafka.use_confluent_cloud:false}") private boolean useConfluent; - @Value("${queue.kafka.confluent.ssl.algorithm}") + @Value("${queue.kafka.confluent.ssl.algorithm:}") private String sslAlgorithm; - @Value("${queue.kafka.confluent.sasl.mechanism}") + @Value("${queue.kafka.confluent.sasl.mechanism:}") private String saslMechanism; - @Value("${queue.kafka.confluent.sasl.config}") + @Value("${queue.kafka.confluent.sasl.config:}") private String saslConfig; - @Value("${queue.kafka.confluent.security.protocol}") + @Value("${queue.kafka.confluent.security.protocol:}") private String securityProtocol; @Setter From e621a21df31475a8fec2b0ce36674e9ec4b23cb0 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 17 Apr 2023 14:51:55 +0200 Subject: [PATCH 03/10] kafka settings: test added, refactored to use configureSSL exact once --- .../server/queue/kafka/TbKafkaSettings.java | 8 +-- .../queue/kafka/TbKafkaSettingsTest.java | 67 +++++++++++++++++++ 2 files changed, 69 insertions(+), 6 deletions(-) create mode 100644 common/queue/src/test/java/org/thingsboard/server/queue/kafka/TbKafkaSettingsTest.java diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index 57be596da7..410c8a649c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -134,8 +134,6 @@ public class TbKafkaSettings { props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, servers); props.put(AdminClientConfig.RETRIES_CONFIG, retries); - configureSSL(props); - return props; } @@ -147,8 +145,6 @@ public class TbKafkaSettings { props.put(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, fetchMaxBytes); props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, maxPollIntervalMs); - configureSSL(props); - props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class); @@ -174,7 +170,7 @@ public class TbKafkaSettings { return props; } - private Properties toProps() { + Properties toProps() { Properties props = new Properties(); if (useConfluent) { @@ -193,7 +189,7 @@ public class TbKafkaSettings { return props; } - private void configureSSL(Properties props) { + void configureSSL(Properties props) { if (sslEnabled) { props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL"); props.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, sslTruststoreLocation); diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/kafka/TbKafkaSettingsTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/kafka/TbKafkaSettingsTest.java new file mode 100644 index 0000000000..48beeb4cd6 --- /dev/null +++ b/common/queue/src/test/java/org/thingsboard/server/queue/kafka/TbKafkaSettingsTest.java @@ -0,0 +1,67 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.kafka; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.spy; + +@SpringBootTest(classes = TbKafkaSettings.class) +@TestPropertySource(properties = { + "queue.type=kafka", + "queue.kafka.bootstrap.servers=localhost:9092", +}) +class TbKafkaSettingsTest { + + @Autowired + TbKafkaSettings settings; + + @BeforeEach + void beforeEach() { + settings = spy(settings); //SpyBean is not aware on @ConditionalOnProperty, that is why the traditional spy in use + } + + @Test + void givenToProps_whenConfigureSSL_thenVerifyOnce() { + settings.toProps(); + Mockito.verify(settings).configureSSL(any()); + } + + @Test + void givenToAdminProps_whenConfigureSSL_thenVerifyOnce() { + settings.toAdminProps(); + Mockito.verify(settings).configureSSL(any()); + } + + @Test + void givenToConsumerProps_whenConfigureSSL_thenVerifyOnce() { + settings.toConsumerProps("main"); + Mockito.verify(settings).configureSSL(any()); + } + + @Test + void givenTotoProducerProps_whenConfigureSSL_thenVerifyOnce() { + settings.toProducerProps(); + Mockito.verify(settings).configureSSL(any()); + } + +} \ No newline at end of file From 26ed90c534dd20378260eef22e3b8778deb403b7 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 18 Apr 2023 10:48:51 +0200 Subject: [PATCH 04/10] kafka settings: kafka yaml parameters added for microservices same as thingsboard.yml . PropertyUtils added, duplicated code refactored to getProps. TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS and TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS moved from other to main properties. Other yaml props is deprecated. other-inline added TB_QUEUE_KAFKA_OTHER_PROPERTIES. Tests added. --- .../src/main/resources/thingsboard.yml | 13 ++-- .../servicebus/TbServiceBusQueueConfigs.java | 27 +++----- .../server/queue/kafka/TbKafkaSettings.java | 16 +++++ .../queue/kafka/TbKafkaTopicConfigs.java | 33 +++------- .../pubsub/TbPubSubSubscriptionSettings.java | 27 +++----- .../server/queue/util/PropertyUtils.java | 40 ++++++++++++ .../queue/kafka/TbKafkaSettingsTest.java | 18 +++++- .../server/queue/util/PropertyUtilsTest.java | 62 +++++++++++++++++++ .../src/main/resources/tb-vc-executor.yml | 13 ++-- .../src/main/resources/tb-coap-transport.yml | 17 +++-- .../src/main/resources/tb-http-transport.yml | 17 +++-- .../src/main/resources/tb-lwm2m-transport.yml | 17 +++-- .../src/main/resources/tb-mqtt-transport.yml | 17 +++-- .../src/main/resources/tb-snmp-transport.yml | 17 +++-- 14 files changed, 235 insertions(+), 99 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java create mode 100644 common/queue/src/test/java/org/thingsboard/server/queue/util/PropertyUtilsTest.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 14bf47301b..1bc8c19a82 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1018,6 +1018,8 @@ queue: max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" + request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" @@ -1036,11 +1038,12 @@ queue: # tb_rule_engine.sq: # - key: max.poll.records # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" - other: # In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside - - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms - value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) - - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms - value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) + other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" + other: # DEPRECATED. In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside + # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + # value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) + # - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + # value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) topic-properties: rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java index 6c56d55d36..5e404bc928 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java @@ -19,10 +19,9 @@ import lombok.Getter; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.queue.util.PropertyUtils; import javax.annotation.PostConstruct; -import java.util.HashMap; import java.util.Map; @Component @@ -55,24 +54,12 @@ public class TbServiceBusQueueConfigs { @PostConstruct private void init() { - coreConfigs = getConfigs(coreProperties); - ruleEngineConfigs = getConfigs(ruleEngineProperties); - transportApiConfigs = getConfigs(transportApiProperties); - notificationsConfigs = getConfigs(notificationsProperties); - jsExecutorConfigs = getConfigs(jsExecutorProperties); - vcConfigs = getConfigs(vcProperties); + coreConfigs = PropertyUtils.getProps(coreProperties); + ruleEngineConfigs = PropertyUtils.getProps(ruleEngineProperties); + transportApiConfigs = PropertyUtils.getProps(transportApiProperties); + notificationsConfigs = PropertyUtils.getProps(notificationsProperties); + jsExecutorConfigs = PropertyUtils.getProps(jsExecutorProperties); + vcConfigs = PropertyUtils.getProps(vcProperties); } - private Map getConfigs(String properties) { - Map configs = new HashMap<>(); - if (StringUtils.isNotEmpty(properties)) { - for (String property : properties.split(";")) { - int delimiterPosition = property.indexOf(":"); - String key = property.substring(0, delimiterPosition); - String value = property.substring(delimiterPosition + 1); - configs.put(key, value); - } - } - return configs; - } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index 410c8a649c..7ed188491b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -32,6 +32,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.TbProperty; +import org.thingsboard.server.queue.util.PropertyUtils; import java.util.Collections; import java.util.List; @@ -108,6 +109,12 @@ public class TbKafkaSettings { @Value("${queue.kafka.fetch_max_bytes:134217728}") private int fetchMaxBytes; + @Value("${queue.kafka.request.timeout.ms:30000}") + private int requestTimeoutMs; + + @Value("${queue.kafka.session.timeout.ms:10000}") + private int sessionTimeoutMs; + @Value("${queue.kafka.use_confluent_cloud:false}") private boolean useConfluent; @@ -123,6 +130,10 @@ public class TbKafkaSettings { @Value("${queue.kafka.confluent.security.protocol:}") private String securityProtocol; + @Value("${queue.kafka.other-inline:}") + private String otherInline; + + @Deprecated @Setter private List other; @@ -180,6 +191,11 @@ public class TbKafkaSettings { props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, securityProtocol); } + props.put(CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG, requestTimeoutMs); + props.put(CommonClientConfigs.SESSION_TIMEOUT_MS_CONFIG, sessionTimeoutMs); + + props.putAll(PropertyUtils.getProps(otherInline)); + if (other != null) { other.forEach(kv -> props.put(kv.getKey(), kv.getValue())); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index 64dd6f2720..9888066ee7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -19,10 +19,9 @@ import lombok.Getter; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.queue.util.PropertyUtils; import javax.annotation.PostConstruct; -import java.util.HashMap; import java.util.Map; @Component @@ -66,29 +65,17 @@ public class TbKafkaTopicConfigs { @PostConstruct private void init() { - coreConfigs = getConfigs(coreProperties); - ruleEngineConfigs = getConfigs(ruleEngineProperties); - transportApiRequestConfigs = getConfigs(transportApiProperties); - transportApiResponseConfigs = getConfigs(transportApiProperties); + coreConfigs = PropertyUtils.getProps(coreProperties); + ruleEngineConfigs = PropertyUtils.getProps(ruleEngineProperties); + transportApiRequestConfigs = PropertyUtils.getProps(transportApiProperties); + transportApiResponseConfigs = PropertyUtils.getProps(transportApiProperties); transportApiResponseConfigs.put(NUM_PARTITIONS_SETTING, "1"); - notificationsConfigs = getConfigs(notificationsProperties); - jsExecutorRequestConfigs = getConfigs(jsExecutorProperties); - jsExecutorResponseConfigs = getConfigs(jsExecutorProperties); + notificationsConfigs = PropertyUtils.getProps(notificationsProperties); + jsExecutorRequestConfigs = PropertyUtils.getProps(jsExecutorProperties); + jsExecutorResponseConfigs = PropertyUtils.getProps(jsExecutorProperties); jsExecutorResponseConfigs.put(NUM_PARTITIONS_SETTING, "1"); - fwUpdatesConfigs = getConfigs(fwUpdatesProperties); - vcConfigs = getConfigs(vcProperties); + fwUpdatesConfigs = PropertyUtils.getProps(fwUpdatesProperties); + vcConfigs = PropertyUtils.getProps(vcProperties); } - private Map getConfigs(String properties) { - Map configs = new HashMap<>(); - if (StringUtils.isNotEmpty(properties)) { - for (String property : properties.split(";")) { - int delimiterPosition = property.indexOf(":"); - String key = property.substring(0, delimiterPosition); - String value = property.substring(delimiterPosition + 1); - configs.put(key, value); - } - } - return configs; - } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java index 0a65c7ca7d..3d7ab85ecc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java @@ -19,10 +19,9 @@ import lombok.Getter; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.queue.util.PropertyUtils; import javax.annotation.PostConstruct; -import java.util.HashMap; import java.util.Map; @Component @@ -56,24 +55,12 @@ public class TbPubSubSubscriptionSettings { @PostConstruct private void init() { - coreSettings = getSettings(coreProperties); - ruleEngineSettings = getSettings(ruleEngineProperties); - transportApiSettings = getSettings(transportApiProperties); - notificationsSettings = getSettings(notificationsProperties); - jsExecutorSettings = getSettings(jsExecutorProperties); - vcSettings = getSettings(vcProperties); + coreSettings = PropertyUtils.getProps(coreProperties); + ruleEngineSettings = PropertyUtils.getProps(ruleEngineProperties); + transportApiSettings = PropertyUtils.getProps(transportApiProperties); + notificationsSettings = PropertyUtils.getProps(notificationsProperties); + jsExecutorSettings = PropertyUtils.getProps(jsExecutorProperties); + vcSettings = PropertyUtils.getProps(vcProperties); } - private Map getSettings(String properties) { - Map configs = new HashMap<>(); - if (StringUtils.isNotEmpty(properties)) { - for (String property : properties.split(";")) { - int delimiterPosition = property.indexOf(":"); - String key = property.substring(0, delimiterPosition); - String value = property.substring(delimiterPosition + 1); - configs.put(key, value); - } - } - return configs; - } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java new file mode 100644 index 0000000000..089d7f2219 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.thingsboard.server.common.data.StringUtils; + +import java.util.HashMap; +import java.util.Map; + +public class PropertyUtils { + + public static Map getProps(String properties) { + Map configs = new HashMap<>(); + if (StringUtils.isNotEmpty(properties)) { + for (String property : properties.split(";")) { + if (StringUtils.isNotEmpty(property)) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String value = property.substring(delimiterPosition + 1); + configs.put(key, value); + } + } + } + return configs; + } + +} diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/kafka/TbKafkaSettingsTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/kafka/TbKafkaSettingsTest.java index 48beeb4cd6..7e8c2d319d 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/kafka/TbKafkaSettingsTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/kafka/TbKafkaSettingsTest.java @@ -22,6 +22,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.TestPropertySource; +import java.util.Properties; + +import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.spy; @@ -29,6 +32,7 @@ import static org.mockito.Mockito.spy; @TestPropertySource(properties = { "queue.type=kafka", "queue.kafka.bootstrap.servers=localhost:9092", + "queue.kafka.other-inline=metrics.recording.level:INFO;metrics.sample.window.ms:30000", }) class TbKafkaSettingsTest { @@ -42,25 +46,37 @@ class TbKafkaSettingsTest { @Test void givenToProps_whenConfigureSSL_thenVerifyOnce() { - settings.toProps(); + Properties props = settings.toProps(); + + assertThat(props).as("TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS").containsEntry("request.timeout.ms", 30000); + assertThat(props).as("TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS").containsEntry("session.timeout.ms", 10000); + + //other-inline + assertThat(props).as("metrics.recording.level").containsEntry("metrics.recording.level", "INFO"); + assertThat(props).as("TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS").containsEntry("metrics.sample.window.ms", "30000"); + + Mockito.verify(settings).toProps(); Mockito.verify(settings).configureSSL(any()); } @Test void givenToAdminProps_whenConfigureSSL_thenVerifyOnce() { settings.toAdminProps(); + Mockito.verify(settings).toProps(); Mockito.verify(settings).configureSSL(any()); } @Test void givenToConsumerProps_whenConfigureSSL_thenVerifyOnce() { settings.toConsumerProps("main"); + Mockito.verify(settings).toProps(); Mockito.verify(settings).configureSSL(any()); } @Test void givenTotoProducerProps_whenConfigureSSL_thenVerifyOnce() { settings.toProducerProps(); + Mockito.verify(settings).toProps(); Mockito.verify(settings).configureSSL(any()); } diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/util/PropertyUtilsTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/util/PropertyUtilsTest.java new file mode 100644 index 0000000000..0484cb314f --- /dev/null +++ b/common/queue/src/test/java/org/thingsboard/server/queue/util/PropertyUtilsTest.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +class PropertyUtilsTest { + + @Test + void givenNullOrEmpty_whenGetConfig_thenEmptyMap() { + assertThat(PropertyUtils.getProps(null)).as("null property").isEmpty(); + assertThat(PropertyUtils.getProps("")).as("empty property").isEmpty(); + assertThat(PropertyUtils.getProps(";")).as("ends with ;").isEmpty(); + } + + @Test + void givenKafkaOtherProperties_whenGetConfig_thenReturnMappedValues() { + assertThat(PropertyUtils.getProps("metrics.recording.level:INFO;metrics.sample.window.ms:30000")) + .as("two pairs") + .isEqualTo(Map.of( + "metrics.recording.level", "INFO", + "metrics.sample.window.ms", "30000" + )); + + assertThat(PropertyUtils.getProps("metrics.recording.level:INFO;metrics.sample.window.ms:30000" + ";")) + .as("two pairs ends with ;") + .isEqualTo(Map.of( + "metrics.recording.level", "INFO", + "metrics.sample.window.ms", "30000" + )); + } + + @Test + void givenKafkaTopicProperties_whenGetConfig_thenReturnMappedValues() { + assertThat(PropertyUtils.getProps("retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1")) + .isEqualTo(Map.of( + "retention.ms", "604800000", + "segment.bytes", "26214400", + "retention.bytes", "1048576000", + "partitions", "1", + "min.insync.replicas", "1" + )); + } + +} diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 2f73ab31f7..352f94e091 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -70,6 +70,8 @@ queue: max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" + request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" @@ -88,11 +90,12 @@ queue: # tb_rule_engine.sq: # - key: max.poll.records # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" - other: # In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside - - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms - value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) - - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms - value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) + other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" + other: # DEPRECATED. In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside + # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + # value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) + # - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + # value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) topic-properties: core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 1bf71fee13..7ea553fe5c 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -167,17 +167,24 @@ queue: max.in.flight.requests.per.connection: "${TB_KAFKA_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION:5}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + max_poll_interval_ms: "${TB_QUEUE_KAFKA_MAX_POLL_INTERVAL_MS:300000}" + max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" + max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" + fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" + request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}" sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" - other: # In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside - - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms - value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) - - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms - value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) + other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" + other: # DEPRECATED. In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside + # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + # value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) + # - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + # value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) topic-properties: rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index caeb1bfc2a..346ec48eae 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -152,17 +152,24 @@ queue: max.in.flight.requests.per.connection: "${TB_KAFKA_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION:5}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + max_poll_interval_ms: "${TB_QUEUE_KAFKA_MAX_POLL_INTERVAL_MS:300000}" + max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" + max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" + fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" + request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}" sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" - other: # In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside - - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms - value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) - - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms - value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) + other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" + other: # DEPRECATED. In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside + # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + # value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) + # - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + # value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) topic-properties: rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index f930836d16..4e8167d89d 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -233,17 +233,24 @@ queue: max.in.flight.requests.per.connection: "${TB_KAFKA_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION:5}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + max_poll_interval_ms: "${TB_QUEUE_KAFKA_MAX_POLL_INTERVAL_MS:300000}" + max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" + max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" + fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" + request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}" sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" - other: # In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside - - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms - value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) - - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms - value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) + other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" + other: # DEPRECATED. In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside + # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + # value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) + # - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + # value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) topic-properties: rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index ec60357764..1e0b1ebcd4 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -182,17 +182,24 @@ queue: max.in.flight.requests.per.connection: "${TB_KAFKA_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION:5}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + max_poll_interval_ms: "${TB_QUEUE_KAFKA_MAX_POLL_INTERVAL_MS:300000}" + max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" + max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" + fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" + request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}" sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" - other: # In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside - - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms - value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) - - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms - value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) + other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" + other: # DEPRECATED. In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside + # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + # value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) + # - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + # value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) topic-properties: rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index 169e464e57..9f086bcbc5 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -128,17 +128,24 @@ queue: max.in.flight.requests.per.connection: "${TB_KAFKA_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION:5}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + max_poll_interval_ms: "${TB_QUEUE_KAFKA_MAX_POLL_INTERVAL_MS:300000}" + max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" + max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" + fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" + request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}" sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" - other: # In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside - - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms - value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) - - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms - value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) + other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" + other: # DEPRECATED. In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside + # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + # value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) + # - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + # value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) topic-properties: rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" From 4ca9935224d685d3c238f54f514351ea2cd53eeb Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Tue, 18 Apr 2023 16:10:35 +0300 Subject: [PATCH 05/10] fixed telemetry/attribute update while bulk import --- .../csv/AbstractBulkImportService.java | 5 +- .../controller/BaseDeviceControllerTest.java | 72 +++++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java index b34ba3ef66..45805c5939 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java @@ -113,8 +113,9 @@ public abstract class AbstractBulkImportService importedEntityInfo = saveEntity(entityData.getFields(), user); E entity = importedEntityInfo.getEntity(); - saveKvs(user, entity, entityData.getKvs()); - + if (request.getMapping().getUpdate() || !importedEntityInfo.isUpdated()) { + saveKvs(user, entity, entityData.getKvs()); + } return importedEntityInfo; }, importedEntityInfo -> { diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java index 9b05b38a22..6ffc216dd3 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.controller; import com.datastax.oss.driver.api.core.uuid.Uuids; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -33,11 +34,13 @@ import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Primary; import org.springframework.test.context.ContextConfiguration; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.SaveDeviceWithCredentialsRequest; import org.thingsboard.server.common.data.SaveOtaPackageInfoRequest; @@ -68,6 +71,8 @@ import org.thingsboard.server.service.gateway_device.GatewayNotificationsService import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; @@ -1392,6 +1397,73 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { Assert.assertEquals(savedCredentials, updatedCredentials); } + @Test + public void testBulkImportDeviceWithUpdateFalse() throws Exception { + String deviceName = "firstDevice"; + String attributeValue = "testValue"; + BulkImportRequest request = new BulkImportRequest(); + request.setFile(String.format("NAME,TYPE,DATA\n%s,%s,%s", deviceName, "thermostat", attributeValue)); + BulkImportRequest.Mapping mapping = new BulkImportRequest.Mapping(); + BulkImportRequest.ColumnMapping name = new BulkImportRequest.ColumnMapping(); + name.setType(BulkImportColumnType.NAME); + BulkImportRequest.ColumnMapping type = new BulkImportRequest.ColumnMapping(); + type.setType(BulkImportColumnType.TYPE); + BulkImportRequest.ColumnMapping attribute = new BulkImportRequest.ColumnMapping(); + attribute.setType(BulkImportColumnType.SERVER_ATTRIBUTE); + attribute.setKey("DATA"); + List columns = new ArrayList<>(); + columns.add(name); + columns.add(type); + columns.add(attribute); + + mapping.setColumns(columns); + mapping.setDelimiter(','); + mapping.setUpdate(true); + mapping.setHeader(true); + request.setMapping(mapping); + + //import device + doPostWithTypedResponse("/api/device/bulk_import", request, new TypeReference<>() {}); + Device savedDevice = doGet("/api/tenant/devices?deviceName=" + deviceName, Device.class); + + //check server attribute value + List> values = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice.getId() + + "/values/attributes/SERVER_SCOPE", new TypeReference<>() { + }); + Map serverAttribute = values.stream().filter(att -> att.get("key").equals("DATA")).findFirst().get(); + Assert.assertEquals(attributeValue, serverAttribute.get("value")); + + //update server attribute value + String newAttributeValue = "testValue2"; + JsonNode content = JacksonUtil.toJsonNode("{\"DATA\": \"" + newAttributeValue + "\"}"); + doPost("/api/plugins/telemetry/" + EntityType.DEVICE.name() + "/" + savedDevice.getUuidId() + "/SERVER_SCOPE", content) + .andExpect(status().isOk()); + + //reimport devices + String deviceName2 = "secondDevice"; + String attributeValue2 = "testValue3"; + + request.setFile(String.format("NAME,TYPE,DATA\n%s,%s,%s\n%s,%s,%s", deviceName, "thermostat", attributeValue, + deviceName2, "thermostat", attributeValue2)); + mapping.setUpdate(false); + doPostWithTypedResponse("/api/device/bulk_import", request, new TypeReference<>() {}); + Device savedDevice2 = doGet("/api/tenant/devices?deviceName=" + deviceName2, Device.class); + + //check attribute value was not changed after reimport + List> values2 = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice.getId() + + "/values/attributes/SERVER_SCOPE", new TypeReference<>() { + }); + Map retrievedServerAttribute2 = values2.stream().filter(att -> att.get("key").equals("DATA")).findFirst().get(); + Assert.assertEquals(newAttributeValue, retrievedServerAttribute2.get("value")); + + //check attribute for second device + List> values3 = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice2.getId() + + "/values/attributes/SERVER_SCOPE", new TypeReference<>() { + }); + Map retrievedServerAttribute3 = values3.stream().filter(att -> att.get("key").equals("DATA")).findFirst().get(); + Assert.assertEquals(attributeValue2, retrievedServerAttribute3.get("value")); + } + private Device createDevice(String name) { Device device = new Device(); device.setName(name); From fd3974d97ec20978be19d119119f0ff9d6cb56ec Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 18 Apr 2023 16:34:54 +0300 Subject: [PATCH 06/10] Monitoring for LwM2M --- monitoring/pom.xml | 15 + .../monitoring/client/Lwm2mClient.java | 171 ++++++++ .../monitoring/config/TransportType.java | 4 +- .../Lwm2mTransportMonitoringConfig.java | 33 ++ .../transport/TransportHealthChecker.java | 5 +- .../transport/TransportMonitoringService.java | 71 +++- .../impl/Lwm2mTransportHealthChecker.java | 72 ++++ .../monitoring/util/ResourceUtils.java | 42 ++ .../main/resources/lwm2m/device_profile.json | 59 +++ .../src/main/resources/lwm2m/models/0.xml | 364 ++++++++++++++++++ .../src/main/resources/lwm2m/models/1.xml | 319 +++++++++++++++ .../src/main/resources/lwm2m/models/2.xml | 83 ++++ .../resources/lwm2m/models/test-model.xml | 45 +++ .../src/main/resources/lwm2m/resource.json | 6 + .../src/main/resources/tb-monitoring.yml | 15 + 15 files changed, 1294 insertions(+), 10 deletions(-) create mode 100644 monitoring/src/main/java/org/thingsboard/monitoring/client/Lwm2mClient.java create mode 100644 monitoring/src/main/java/org/thingsboard/monitoring/config/service/Lwm2mTransportMonitoringConfig.java create mode 100644 monitoring/src/main/java/org/thingsboard/monitoring/transport/impl/Lwm2mTransportHealthChecker.java create mode 100644 monitoring/src/main/java/org/thingsboard/monitoring/util/ResourceUtils.java create mode 100644 monitoring/src/main/resources/lwm2m/device_profile.json create mode 100644 monitoring/src/main/resources/lwm2m/models/0.xml create mode 100644 monitoring/src/main/resources/lwm2m/models/1.xml create mode 100644 monitoring/src/main/resources/lwm2m/models/2.xml create mode 100644 monitoring/src/main/resources/lwm2m/models/test-model.xml create mode 100644 monitoring/src/main/resources/lwm2m/resource.json diff --git a/monitoring/pom.xml b/monitoring/pom.xml index 429df521b6..e044635e98 100644 --- a/monitoring/pom.xml +++ b/monitoring/pom.xml @@ -41,6 +41,9 @@ ${project.build.directory}/windows ThingsBoard Monitoring Service org.thingsboard.monitoring.ThingsboardMonitoringApplication + + 2.6.1 + 2.0.0-M4 @@ -65,10 +68,12 @@ org.eclipse.californium californium-core + ${californium.version} org.eclipse.californium scandium + ${californium.version} org.eclipse.paho @@ -78,6 +83,16 @@ org.apache.httpcomponents httpclient + + org.eclipse.leshan + leshan-client-cf + ${leshan.version} + + + org.eclipse.leshan + leshan-core + ${leshan.version} + org.java-websocket Java-WebSocket diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/client/Lwm2mClient.java b/monitoring/src/main/java/org/thingsboard/monitoring/client/Lwm2mClient.java new file mode 100644 index 0000000000..473a17aa00 --- /dev/null +++ b/monitoring/src/main/java/org/thingsboard/monitoring/client/Lwm2mClient.java @@ -0,0 +1,171 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.monitoring.client; + +import lombok.Getter; +import lombok.Setter; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.eclipse.californium.core.network.CoapEndpoint; +import org.eclipse.californium.core.network.config.NetworkConfig; +import org.eclipse.californium.core.observe.ObservationStore; +import org.eclipse.californium.scandium.DTLSConnector; +import org.eclipse.californium.scandium.config.DtlsConnectorConfig; +import org.eclipse.leshan.client.californium.LeshanClient; +import org.eclipse.leshan.client.californium.LeshanClientBuilder; +import org.eclipse.leshan.client.engine.DefaultRegistrationEngineFactory; +import org.eclipse.leshan.client.object.Security; +import org.eclipse.leshan.client.object.Server; +import org.eclipse.leshan.client.resource.BaseInstanceEnabler; +import org.eclipse.leshan.client.resource.DummyInstanceEnabler; +import org.eclipse.leshan.client.resource.ObjectsInitializer; +import org.eclipse.leshan.client.servers.ServerIdentity; +import org.eclipse.leshan.core.californium.EndpointFactory; +import org.eclipse.leshan.core.model.InvalidDDFFileException; +import org.eclipse.leshan.core.model.LwM2mModel; +import org.eclipse.leshan.core.model.ObjectLoader; +import org.eclipse.leshan.core.model.ObjectModel; +import org.eclipse.leshan.core.model.StaticModel; +import org.eclipse.leshan.core.node.codec.DefaultLwM2mDecoder; +import org.eclipse.leshan.core.node.codec.DefaultLwM2mEncoder; +import org.eclipse.leshan.core.response.ReadResponse; +import org.thingsboard.monitoring.util.ResourceUtils; + +import javax.security.auth.Destroyable; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.eclipse.leshan.client.object.Security.noSec; +import static org.eclipse.leshan.core.LwM2mId.ACCESS_CONTROL; +import static org.eclipse.leshan.core.LwM2mId.DEVICE; +import static org.eclipse.leshan.core.LwM2mId.SECURITY; +import static org.eclipse.leshan.core.LwM2mId.SERVER; + +@Slf4j +public class Lwm2mClient extends BaseInstanceEnabler implements Destroyable { + + @Getter + @Setter + private LeshanClient leshanClient; + + private static final List supportedResources = Collections.singletonList(0); + + private String data = ""; + + private String serverUri; + private String endpoint; + + public Lwm2mClient(String serverUri, String endpoint) { + this.serverUri = serverUri; + this.endpoint = endpoint; + } + + public Lwm2mClient() { + } + + public void initClient() throws InvalidDDFFileException, IOException { + String[] resources = new String[]{"0.xml", "1.xml", "2.xml", "test-model.xml"}; + List models = new ArrayList<>(); + for (String resourceName : resources) { + models.addAll(ObjectLoader.loadDdfFile(ResourceUtils.getResourceAsStream("lwm2m/models/" + resourceName), resourceName)); + } + + Security security = noSec(serverUri, 123); + NetworkConfig coapConfig = new NetworkConfig().setString(NetworkConfig.Keys.COAP_PORT, StringUtils.substringAfterLast(serverUri, ":")); + + LeshanClient leshanClient; + + LwM2mModel model = new StaticModel(models); + ObjectsInitializer initializer = new ObjectsInitializer(model); + initializer.setInstancesForObject(SECURITY, security); + initializer.setInstancesForObject(SERVER, new Server(123, 300)); + initializer.setInstancesForObject(DEVICE, this); + initializer.setClassForObject(ACCESS_CONTROL, DummyInstanceEnabler.class); + DtlsConnectorConfig.Builder dtlsConfig = new DtlsConnectorConfig.Builder(); + dtlsConfig.setRecommendedCipherSuitesOnly(true); + dtlsConfig.setClientOnly(); + + DefaultRegistrationEngineFactory engineFactory = new DefaultRegistrationEngineFactory(); + engineFactory.setReconnectOnUpdate(false); + engineFactory.setResumeOnConnect(true); + + EndpointFactory endpointFactory = new EndpointFactory() { + + @Override + public CoapEndpoint createUnsecuredEndpoint(InetSocketAddress address, NetworkConfig coapConfig, + ObservationStore store) { + CoapEndpoint.Builder builder = new CoapEndpoint.Builder(); + builder.setInetSocketAddress(address); + builder.setNetworkConfig(coapConfig); + return builder.build(); + } + + @Override + public CoapEndpoint createSecuredEndpoint(DtlsConnectorConfig dtlsConfig, NetworkConfig coapConfig, + ObservationStore store) { + CoapEndpoint.Builder builder = new CoapEndpoint.Builder(); + DtlsConnectorConfig.Builder dtlsConfigBuilder = new DtlsConnectorConfig.Builder(dtlsConfig); + builder.setConnector(new DTLSConnector(dtlsConfigBuilder.build())); + builder.setNetworkConfig(coapConfig); + return builder.build(); + } + }; + + LeshanClientBuilder builder = new LeshanClientBuilder(endpoint); + builder.setObjects(initializer.createAll()); + builder.setCoapConfig(coapConfig); + builder.setDtlsConfig(dtlsConfig); + builder.setRegistrationEngineFactory(engineFactory); + builder.setEndpointFactory(endpointFactory); + builder.setDecoder(new DefaultLwM2mDecoder(false)); + builder.setEncoder(new DefaultLwM2mEncoder(false)); + leshanClient = builder.build(); + + setLeshanClient(leshanClient); + + leshanClient.start(); + } + + @Override + public List getAvailableResourceIds(ObjectModel model) { + return supportedResources; + } + + @Override + public ReadResponse read(ServerIdentity identity, int resourceId) { + if (supportedResources.contains(resourceId)) { + return ReadResponse.success(resourceId, data); + } + return super.read(identity, resourceId); + } + + @SneakyThrows + public void send(String data, int resource) { + this.data = data; + fireResourcesChange(resource); + } + + @Override + public void destroy() { + if (leshanClient != null) { + leshanClient.destroy(true); + } + } +} diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/config/TransportType.java b/monitoring/src/main/java/org/thingsboard/monitoring/config/TransportType.java index ed0a4ea331..a3aaa7ec98 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/config/TransportType.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/config/TransportType.java @@ -20,6 +20,7 @@ import lombok.Getter; import org.thingsboard.monitoring.transport.TransportHealthChecker; import org.thingsboard.monitoring.transport.impl.CoapTransportHealthChecker; import org.thingsboard.monitoring.transport.impl.HttpTransportHealthChecker; +import org.thingsboard.monitoring.transport.impl.Lwm2mTransportHealthChecker; import org.thingsboard.monitoring.transport.impl.MqttTransportHealthChecker; @AllArgsConstructor @@ -28,7 +29,8 @@ public enum TransportType { MQTT(MqttTransportHealthChecker.class), COAP(CoapTransportHealthChecker.class), - HTTP(HttpTransportHealthChecker.class); + HTTP(HttpTransportHealthChecker.class), + LWM2M(Lwm2mTransportHealthChecker.class); private final Class> serviceClass; diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/config/service/Lwm2mTransportMonitoringConfig.java b/monitoring/src/main/java/org/thingsboard/monitoring/config/service/Lwm2mTransportMonitoringConfig.java new file mode 100644 index 0000000000..4d97de8366 --- /dev/null +++ b/monitoring/src/main/java/org/thingsboard/monitoring/config/service/Lwm2mTransportMonitoringConfig.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.monitoring.config.service; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; +import org.thingsboard.monitoring.config.TransportType; + +@Component +@ConditionalOnProperty(name = "monitoring.transports.lwm2m.enabled", havingValue = "true") +@ConfigurationProperties(prefix = "monitoring.transports.lwm2m") +public class Lwm2mTransportMonitoringConfig extends TransportMonitoringConfig { + + @Override + public TransportType getTransportType() { + return TransportType.LWM2M; + } + +} diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/transport/TransportHealthChecker.java b/monitoring/src/main/java/org/thingsboard/monitoring/transport/TransportHealthChecker.java index 9717b98f51..0c742e7709 100644 --- a/monitoring/src/main/java/org/thingsboard/monitoring/transport/TransportHealthChecker.java +++ b/monitoring/src/main/java/org/thingsboard/monitoring/transport/TransportHealthChecker.java @@ -67,7 +67,7 @@ public abstract class TransportHealthChecker { log.info("Creating new device '{}'", deviceName); - Device monitoringDevice = new Device(); - monitoringDevice.setName(deviceName); - monitoringDevice.setType("default"); - DeviceData deviceData = new DeviceData(); - deviceData.setConfiguration(new DefaultDeviceConfiguration()); - deviceData.setTransportConfiguration(new DefaultDeviceTransportConfiguration()); - return tbClient.saveDevice(monitoringDevice); + return createDevice(config.getTransportType(), deviceName, tbClient); }); deviceId = device.getId(); target.getDevice().setId(deviceId.toString()); @@ -134,10 +141,58 @@ public final class TransportMonitoringService { deviceId = new DeviceId(deviceConfig.getId()); } - log.info("Loading credentials for device {}", deviceId); + log.info("Using device {} for {} monitoring", deviceId, config.getTransportType()); DeviceCredentials credentials = tbClient.getDeviceCredentialsByDeviceId(deviceId) .orElseThrow(() -> new IllegalArgumentException("No credentials found for device " + deviceId)); target.getDevice().setCredentials(credentials); } + private Device createDevice(TransportType transportType, String name, TbClient tbClient) { + Device device = new Device(); + device.setName(name); + + DeviceCredentials credentials = new DeviceCredentials(); + credentials.setCredentialsId(RandomStringUtils.randomAlphabetic(20)); + + DeviceData deviceData = new DeviceData(); + deviceData.setConfiguration(new DefaultDeviceConfiguration()); + if (transportType != TransportType.LWM2M) { + device.setType("default"); + deviceData.setTransportConfiguration(new DefaultDeviceTransportConfiguration()); + credentials.setCredentialsType(DeviceCredentialsType.ACCESS_TOKEN); + } else { + tbClient.getResources(new PageLink(1, 0, "lwm2m monitoring")).getData() + .stream().findFirst() + .orElseGet(() -> { + TbResource newResource = ResourceUtils.getResource("lwm2m/resource.json", TbResource.class); + log.info("Creating LwM2M resource"); + return tbClient.saveResource(newResource); + }); + String profileName = "LwM2M Monitoring"; + DeviceProfile profile = tbClient.getDeviceProfiles(new PageLink(1, 0, profileName)).getData() + .stream().findFirst() + .orElseGet(() -> { + DeviceProfile newProfile = ResourceUtils.getResource("lwm2m/device_profile.json", DeviceProfile.class); + newProfile.setName(profileName); + log.info("Creating LwM2M device profile"); + return tbClient.saveDeviceProfile(newProfile); + }); + device.setType(profileName); + device.setDeviceProfileId(profile.getId()); + deviceData.setTransportConfiguration(new Lwm2mDeviceTransportConfiguration()); + + credentials.setCredentialsType(DeviceCredentialsType.LWM2M_CREDENTIALS); + LwM2MDeviceCredentials lwm2mCreds = new LwM2MDeviceCredentials(); + NoSecClientCredential client = new NoSecClientCredential(); + client.setEndpoint(credentials.getCredentialsId()); + lwm2mCreds.setClient(client); + LwM2MBootstrapClientCredentials bootstrap = new LwM2MBootstrapClientCredentials(); + bootstrap.setBootstrapServer(new NoSecBootstrapClientCredential()); + bootstrap.setLwm2mServer(new NoSecBootstrapClientCredential()); + lwm2mCreds.setBootstrap(bootstrap); + credentials.setCredentialsValue(JacksonUtil.toString(lwm2mCreds)); + } + return tbClient.saveDeviceWithCredentials(device, credentials).get(); + } + } diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/transport/impl/Lwm2mTransportHealthChecker.java b/monitoring/src/main/java/org/thingsboard/monitoring/transport/impl/Lwm2mTransportHealthChecker.java new file mode 100644 index 0000000000..6487003e0c --- /dev/null +++ b/monitoring/src/main/java/org/thingsboard/monitoring/transport/impl/Lwm2mTransportHealthChecker.java @@ -0,0 +1,72 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.monitoring.transport.impl; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Service; +import org.thingsboard.monitoring.client.Lwm2mClient; +import org.thingsboard.monitoring.config.MonitoringTargetConfig; +import org.thingsboard.monitoring.config.TransportType; +import org.thingsboard.monitoring.config.service.Lwm2mTransportMonitoringConfig; +import org.thingsboard.monitoring.transport.TransportHealthChecker; + +@Service +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +@Slf4j +public class Lwm2mTransportHealthChecker extends TransportHealthChecker { + + private Lwm2mClient lwm2mClient; + + protected Lwm2mTransportHealthChecker(Lwm2mTransportMonitoringConfig config, MonitoringTargetConfig target) { + super(config, target); + } + + @Override + protected void initClient() throws Exception { + if (lwm2mClient == null || lwm2mClient.getLeshanClient() == null || lwm2mClient.isDestroyed()) { + String endpoint = target.getDevice().getCredentials().getCredentialsId(); + lwm2mClient = new Lwm2mClient(target.getBaseUrl(), endpoint); + lwm2mClient.initClient(); + log.debug("Initialized LwM2M client for endpoint '{}'", endpoint); + } + } + + @Override + protected void sendTestPayload(String payload) throws Exception { + lwm2mClient.send(payload, 0); + } + + @Override + protected String createTestPayload(String testValue) { + return testValue; + } + + @Override + protected void destroyClient() throws Exception { + if (lwm2mClient != null) { + lwm2mClient.destroy(); + lwm2mClient = null; + } + } + + @Override + protected TransportType getTransportType() { + return TransportType.LWM2M; + } + +} diff --git a/monitoring/src/main/java/org/thingsboard/monitoring/util/ResourceUtils.java b/monitoring/src/main/java/org/thingsboard/monitoring/util/ResourceUtils.java new file mode 100644 index 0000000000..4cc1efe8cb --- /dev/null +++ b/monitoring/src/main/java/org/thingsboard/monitoring/util/ResourceUtils.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.monitoring.util; + +import lombok.SneakyThrows; +import org.thingsboard.common.util.JacksonUtil; + +import java.io.InputStream; + +public class ResourceUtils { + + @SneakyThrows + public static T getResource(String path, Class type) { + InputStream resource = ResourceUtils.class.getClassLoader().getResourceAsStream(path); + if (resource == null) { + throw new IllegalArgumentException("Resource not found for path " + path); + } + return JacksonUtil.OBJECT_MAPPER.readValue(resource, type); + } + + public static InputStream getResourceAsStream(String path) { + InputStream resource = ResourceUtils.class.getClassLoader().getResourceAsStream(path); + if (resource == null) { + throw new IllegalArgumentException("Resource not found for path " + path); + } + return resource; + } + +} diff --git a/monitoring/src/main/resources/lwm2m/device_profile.json b/monitoring/src/main/resources/lwm2m/device_profile.json new file mode 100644 index 0000000000..7f93a7e6b6 --- /dev/null +++ b/monitoring/src/main/resources/lwm2m/device_profile.json @@ -0,0 +1,59 @@ +{ + "name": "LwM2M Monitoring", + "type": "DEFAULT", + "image": null, + "defaultQueueName": null, + "transportType": "LWM2M", + "provisionType": "DISABLED", + "description": "", + "profileData": { + "configuration": { + "type": "DEFAULT" + }, + "transportConfiguration": { + "observeAttr": { + "observe": [ + "/3_1.0/0/0" + ], + "attribute": [], + "telemetry": [ + "/3_1.0/0/0" + ], + "keyName": { + "/3_1.0/0/0": "testData" + }, + "attributeLwm2m": {} + }, + "bootstrap": [ + { + "shortServerId": 123, + "bootstrapServerIs": false, + "host": "0.0.0.0", + "port": 5685, + "clientHoldOffTime": 1, + "serverPublicKey": "", + "serverCertificate": "", + "bootstrapServerAccountTimeout": 0, + "lifetime": 300, + "defaultMinPeriod": 1, + "notifIfDisabled": true, + "binding": "U", + "securityMode": "NO_SEC" + } + ], + "clientLwM2mSettings": { + "clientOnlyObserveAfterConnect": 1, + "fwUpdateStrategy": 1, + "swUpdateStrategy": 1, + "powerMode": "DRX", + "compositeOperationsSupport": false + }, + "bootstrapServerUpdateEnable": false, + "type": "LWM2M" + }, + "alarms": null, + "provisionConfiguration": { + "type": "DISABLED" + } + } +} \ No newline at end of file diff --git a/monitoring/src/main/resources/lwm2m/models/0.xml b/monitoring/src/main/resources/lwm2m/models/0.xml new file mode 100644 index 0000000000..d122984c16 --- /dev/null +++ b/monitoring/src/main/resources/lwm2m/models/0.xml @@ -0,0 +1,364 @@ + + + + + LWM2M Security + + 0 + urn:oma:lwm2m:oma:0:1.2 + 1.1 + 1.2 + Multiple + Mandatory + + + LWM2M Server URI + + Single + Mandatory + String + 0..255 + + + + + Bootstrap-Server + + Single + Mandatory + Boolean + + + + + + Security Mode + + Single + Mandatory + Integer + 0..4 + + + + + Public Key or Identity + + Single + Mandatory + Opaque + + + + + + Server Public Key + + Single + Mandatory + Opaque + + + + + + Secret Key + + Single + Mandatory + Opaque + + + + + + SMS Security Mode + + Single + Optional + Integer + 0..255 + + + + + SMS Binding Key Parameters + + Single + Optional + Opaque + 6 + + + + + SMS Binding Secret Key(s) + + Single + Optional + Opaque + 16,32,48 + + + + + LwM2M Server SMS Number + + Single + Optional + String + + + + + + Short Server ID + + Single + Optional + Integer + 1..65534 + + + + + Client Hold Off Time + + Single + Optional + Integer + + s + + + + Bootstrap-Server Account Timeout + + Single + Optional + Integer + + s + + + + Matching Type + + Single + Optional + Integer + 0..3 + + + + + SNI + + Single + Optional + String + + + + + + Certificate Usage + + Single + Optional + Integer + 0..3 + + + + + DTLS/TLS Ciphersuite + + Multiple + Optional + Integer + + + + + OSCORE Security Mode + + Single + Optional + Objlnk + + + + + + Groups To Use by Client + + Multiple + Optional + Integer + 0..65535 + + + + + Signature Algorithms Supported by Server + + Multiple + Optional + Integer + 0..65535 + + + + Signature Algorithms To Use by Client + + Multiple + Optional + Integer + 0..65535 + + + + + Signature Algorithm Certs Supported by Server + + Multiple + Optional + Integer + 0..65535 + + + + + TLS 1.3 Features To Use by Client + + Single + Optional + Integer + 0..65535 + + + + + TLS Extensions Supported by Server + + Single + Optional + Integer + 0..65535 + + + + + TLS Extensions To Use by Client + + Single + Optional + Integer + 0..65535 + + + + + Secondary LwM2M Server URI + + Multiple + Optional + String + 0..255 + + + + MQTT Server + + Single + Optional + Objlnk + + + + + LwM2M COSE Security + + Multiple + Optional + Objlnk + + + + + RDS Destination Port + + Single + Optional + Integer + 0..15 + + + + RDS Source Port + + Single + Optional + Integer + 0..15 + + + + RDS Application ID + + Single + Optional + String + + + + + + + + diff --git a/monitoring/src/main/resources/lwm2m/models/1.xml b/monitoring/src/main/resources/lwm2m/models/1.xml new file mode 100644 index 0000000000..a81caa27b0 --- /dev/null +++ b/monitoring/src/main/resources/lwm2m/models/1.xml @@ -0,0 +1,319 @@ + + + + + LwM2M Server + + 1 + urn:oma:lwm2m:oma:1:1.2 + 1.2 + 1.2 + Multiple + Mandatory + + + Short Server ID + R + Single + Mandatory + Integer + 1..65534 + + + + + Lifetime + RW + Single + Mandatory + Integer + + s + + + + Default Minimum Period + RW + Single + Optional + Integer + + s + + + + Default Maximum Period + RW + Single + Optional + Integer + + s + + + + Disable + E + Single + Optional + + + + + + + Disable Timeout + RW + Single + Optional + Integer + + s + + + + Notification Storing When Disabled or Offline + RW + Single + Mandatory + Boolean + + + + + + Binding + RW + Single + Mandatory + String + + + + + + Registration Update Trigger + E + Single + Mandatory + + + + + + + Bootstrap-Request Trigger + E + Single + Optional + + + + + + + APN Link + RW + Single + Optional + Objlnk + + + + + + TLS-DTLS Alert Code + R + Single + Optional + Integer + 0..255 + + + + + Last Bootstrapped + R + Single + Optional + Time + + + + + + Registration Priority Order + R + Single + Optional + Integer + + + + + + Initial Registration Delay Timer + RW + Single + Optional + Integer + + s + + + + Registration Failure Block + R + Single + Optional + Boolean + + + + + + Bootstrap on Registration Failure + R + Single + Optional + Boolean + + + + + + Communication Retry Count + RW + Single + Optional + Integer + + + + + + Communication Retry Timer + RW + Single + Optional + Integer + + s + + + + Communication Sequence Delay Timer + RW + Single + Optional + Integer + + s + + + + Communication Sequence Retry Count + RW + Single + Optional + Integer + + + + + + Trigger + RW + Single + Optional + Boolean + + + + + + Preferred Transport + RW + Single + Optional + String + The possible values are those listed in the LwM2M Core Specification + + + + Mute Send + RW + Single + Optional + Boolean + + + + + + Alternate APN Links + RW + Multiple + Optional + Objlnk + + + + + + Supported Server Versions + RW + Multiple + Optional + String + + + + + + Default Notification Mode + RW + Single + Optional + Integer + 0..1 + + + + + Profile ID Hash Algorithm + RW + Single + Optional + Integer + 0..255 + + + + + + + diff --git a/monitoring/src/main/resources/lwm2m/models/2.xml b/monitoring/src/main/resources/lwm2m/models/2.xml new file mode 100644 index 0000000000..79b2ed1321 --- /dev/null +++ b/monitoring/src/main/resources/lwm2m/models/2.xml @@ -0,0 +1,83 @@ + + + + + LwM2M Access Control + + 2 + urn:oma:lwm2m:oma:2:1.1 + 1.0 + 1.1 + Multiple + Optional + + + Object ID + R + Single + Mandatory + Integer + 1..65534 + + + + + Object Instance ID + R + Single + Mandatory + Integer + 0..65535 + + + + + ACL + RW + Multiple + Optional + Integer + 0..31 + + + + + Access Control Owner + RW + Single + Mandatory + Integer + 0..65535 + + + + + + + diff --git a/monitoring/src/main/resources/lwm2m/models/test-model.xml b/monitoring/src/main/resources/lwm2m/models/test-model.xml new file mode 100644 index 0000000000..c8e49ce32c --- /dev/null +++ b/monitoring/src/main/resources/lwm2m/models/test-model.xml @@ -0,0 +1,45 @@ + + + + + LwM2M Monitoring + + + 3 + urn:oma:lwm2m:oma:3:1.0 + 1.1 + 1.0 + Single + Mandatory + + + Test data + R + Single + Optional + String + + + + + + + + diff --git a/monitoring/src/main/resources/lwm2m/resource.json b/monitoring/src/main/resources/lwm2m/resource.json new file mode 100644 index 0000000000..b624fa400e --- /dev/null +++ b/monitoring/src/main/resources/lwm2m/resource.json @@ -0,0 +1,6 @@ +{ + "title": "", + "resourceType": "LWM2M_MODEL", + "fileName": "test-model.xml", + "data": "PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPCEtLQoKICAgIENvcHlyaWdodCDCqSAyMDE2LTIwMjIgVGhlIFRoaW5nc2JvYXJkIEF1dGhvcnMKCiAgICBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKICAgIHlvdSBtYXkgbm90IHVzZSB0aGlzIGZpbGUgZXhjZXB0IGluIGNvbXBsaWFuY2Ugd2l0aCB0aGUgTGljZW5zZS4KICAgIFlvdSBtYXkgb2J0YWluIGEgY29weSBvZiB0aGUgTGljZW5zZSBhdAoKICAgICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKCiAgICBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiAgICBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAogICAgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiAgICBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiAgICBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi0tPgo8TFdNMk0geG1sbnM6eHNpPSJodHRwOi8vd3d3LnczLm9yZy8yMDAxL1hNTFNjaGVtYS1pbnN0YW5jZSIKICAgICAgIHhzaTpub05hbWVzcGFjZVNjaGVtYUxvY2F0aW9uPSJodHRwOi8vd3d3Lm9wZW5tb2JpbGVhbGxpYW5jZS5vcmcvdGVjaC9wcm9maWxlcy9MV00yTS12MV8xLnhzZCI+CiAgICA8T2JqZWN0IE9iamVjdFR5cGU9Ik1PRGVmaW5pdGlvbiI+CiAgICAgICAgPE5hbWU+THdNMk0gTW9uaXRvcmluZzwvTmFtZT4KICAgICAgICA8RGVzY3JpcHRpb24xPgogICAgICAgICAgICA8IVtDREFUQVtdXT48L0Rlc2NyaXB0aW9uMT4KICAgICAgICA8T2JqZWN0SUQ+MzwvT2JqZWN0SUQ+CiAgICAgICAgPE9iamVjdFVSTj51cm46b21hOmx3bTJtOm9tYTozOjEuMDwvT2JqZWN0VVJOPgogICAgICAgIDxMV00yTVZlcnNpb24+MS4xPC9MV00yTVZlcnNpb24+CiAgICAgICAgPE9iamVjdFZlcnNpb24+MS4wPC9PYmplY3RWZXJzaW9uPgogICAgICAgIDxNdWx0aXBsZUluc3RhbmNlcz5TaW5nbGU8L011bHRpcGxlSW5zdGFuY2VzPgogICAgICAgIDxNYW5kYXRvcnk+TWFuZGF0b3J5PC9NYW5kYXRvcnk+CiAgICAgICAgPFJlc291cmNlcz4KICAgICAgICAgICAgPEl0ZW0gSUQ9IjAiPgogICAgICAgICAgICAgICAgPE5hbWU+VGVzdCBkYXRhPC9OYW1lPgogICAgICAgICAgICAgICAgPE9wZXJhdGlvbnM+UjwvT3BlcmF0aW9ucz4KICAgICAgICAgICAgICAgIDxNdWx0aXBsZUluc3RhbmNlcz5TaW5nbGU8L011bHRpcGxlSW5zdGFuY2VzPgogICAgICAgICAgICAgICAgPE1hbmRhdG9yeT5PcHRpb25hbDwvTWFuZGF0b3J5PgogICAgICAgICAgICAgICAgPFR5cGU+U3RyaW5nPC9UeXBlPgogICAgICAgICAgICAgICAgPFJhbmdlRW51bWVyYXRpb24+PC9SYW5nZUVudW1lcmF0aW9uPgogICAgICAgICAgICAgICAgPFVuaXRzPjwvVW5pdHM+CiAgICAgICAgICAgICAgICA8RGVzY3JpcHRpb24+PCFbQ0RBVEFbVGVzdCBkYXRhXV0+PC9EZXNjcmlwdGlvbj4KICAgICAgICAgICAgPC9JdGVtPgogICAgICAgIDwvUmVzb3VyY2VzPgogICAgICAgIDxEZXNjcmlwdGlvbjI+PC9EZXNjcmlwdGlvbjI+CiAgICA8L09iamVjdD4KPC9MV00yTT4K" +} \ No newline at end of file diff --git a/monitoring/src/main/resources/tb-monitoring.yml b/monitoring/src/main/resources/tb-monitoring.yml index 403b29acc2..d89886cd72 100644 --- a/monitoring/src/main/resources/tb-monitoring.yml +++ b/monitoring/src/main/resources/tb-monitoring.yml @@ -91,6 +91,21 @@ monitoring: # monitoring.transports.http.targets[1].base_url, monitoring.transports.http.targets[1].device.id, # monitoring.transports.http.targets[2].base_url, monitoring.transports.http.targets[2].device.id, etc. + lwm2m: + # Enable LwM2M checks + enabled: '${LWM2M_TRANSPORT_MONITORING_ENABLED:true}' + # LwM2M request timeout in milliseconds + request_timeout_ms: '${LWM2M_REQUEST_TIMEOUT_MS:4000}' + targets: + # LwM2M base url, coap://DOMAIN:5685 by default + - base_url: '${LWM2M_TRANSPORT_BASE_URL:coap://${monitoring.domain}:5685}' + # LwM2M device to push telemetry for. If not set - device will be found or created automatically + device: + id: '${LWM2M_TRANSPORT_TARGET_DEVICE_ID:}' + # To add more targets, use following environment variables: + # monitoring.transports.lwm2m.targets[1].base_url, monitoring.transports.lwm2m.targets[1].device.id, + # monitoring.transports.lwm2m.targets[2].base_url, monitoring.transports.lwm2m.targets[2].device.id, etc. + notification_channels: slack: # Enable notifying via Slack From f3402ace63f181fe44df8bb9e73ba54d529ecbb2 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 18 Apr 2023 19:05:38 +0200 Subject: [PATCH 07/10] features info improvements --- .../system/DefaultSystemInfoService.java | 44 ++++++++++++----- .../controller/BaseHomePageApiTest.java | 48 +++++++++++++++++-- 2 files changed, 77 insertions(+), 15 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java index cce38cfd18..80d4cede03 100644 --- a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java +++ b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.system; -import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.FutureCallback; import com.google.protobuf.ProtocolStringList; import lombok.RequiredArgsConstructor; @@ -23,10 +22,11 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.rule.engine.api.MailService; +import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.FeaturesInfo; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.SystemInfo; import org.thingsboard.server.common.data.SystemInfoData; import org.thingsboard.server.common.data.id.TenantId; @@ -38,7 +38,6 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.stats.TbApiUsageStateClient; import org.thingsboard.server.dao.oauth2.OAuth2Service; -import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; import org.thingsboard.server.queue.discovery.DiscoveryService; @@ -59,10 +58,10 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import static org.thingsboard.common.util.SystemUtil.getCpuCount; import static org.thingsboard.common.util.SystemUtil.getCpuUsage; -import static org.thingsboard.common.util.SystemUtil.getMemoryUsage; import static org.thingsboard.common.util.SystemUtil.getDiscSpaceUsage; -import static org.thingsboard.common.util.SystemUtil.getCpuCount; +import static org.thingsboard.common.util.SystemUtil.getMemoryUsage; import static org.thingsboard.common.util.SystemUtil.getTotalDiscSpace; import static org.thingsboard.common.util.SystemUtil.getTotalMemory; @@ -90,6 +89,8 @@ public class DefaultSystemInfoService extends TbApplicationEventListener".equalsIgnoreCase(mailFrom.asText()); + try { + mailService.testConnection(TenantId.SYS_TENANT_ID); + return true; + } catch (Exception e) { + return false; + } + } + + private boolean isTwoFaEnabled() { + AdminSettings twoFaSettings = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "twoFaSettings"); + if (twoFaSettings != null) { + var providers = twoFaSettings.getJsonValue().get("providers"); + if (providers != null) { + return providers.size() > 0; + } + } + return false; + } + + private boolean isSlackEnabled() { + AdminSettings notifications = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "notifications"); + if (notifications != null) { + return notifications.getJsonValue().get("deliveryMethodsConfigs").has("SLACK"); } return false; } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseHomePageApiTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseHomePageApiTest.java index 6dec1f0313..a02c867948 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseHomePageApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseHomePageApiTest.java @@ -21,8 +21,12 @@ import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.api.MailService; +import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.Customer; @@ -53,7 +57,9 @@ import org.thingsboard.server.common.data.query.TsValue; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.stats.TbApiUsageStateClient; +import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; +import org.thingsboard.server.service.security.auth.mfa.config.TwoFaConfigManager; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityCountCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityCountUpdate; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityDataUpdate; @@ -76,6 +82,18 @@ public abstract class BaseHomePageApiTest extends AbstractControllerTest { @Autowired private TbTenantProfileCache tenantProfileCache; + @Autowired + private AdminSettingsService adminSettingsService; + + @MockBean + private MailService mailService; + + @MockBean + private SmsService smsService; + + @MockBean + TwoFaConfigManager twoFaConfigManager; + //For system administrator @Test public void testTenantsCountWsCmd() throws Exception { @@ -266,6 +284,19 @@ public abstract class BaseHomePageApiTest extends AbstractControllerTest { @Test public void testGetFeaturesInfo() throws Exception { + String mail = "test@thingsboard.org"; + Mockito.doAnswer(invocation -> { + AdminSettings mailSettings = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "mail"); + JsonNode mailFrom = mailSettings.getJsonValue().get("mailFrom"); + if (!mailFrom.asText().equals(mail)) { + throw new Exception(); + } + return null; + }).when(mailService).testConnection(TenantId.SYS_TENANT_ID); + + Mockito.when(smsService.isConfigured(TenantId.SYS_TENANT_ID)) + .then(a -> adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "sms") != null); + loginSysAdmin(); FeaturesInfo featuresInfo = doGet("/api/admin/featuresInfo", FeaturesInfo.class); @@ -279,7 +310,7 @@ public abstract class BaseHomePageApiTest extends AbstractControllerTest { AdminSettings mailSettings = doGet("/api/admin/settings/mail", AdminSettings.class); JsonNode jsonValue = mailSettings.getJsonValue(); - ((ObjectNode) jsonValue).put("mailFrom", "test@thingsboard.org"); + ((ObjectNode) jsonValue).put("mailFrom", mail); mailSettings.setJsonValue(jsonValue); doPost("/api/admin/settings", mailSettings).andExpect(status().isOk()); @@ -305,7 +336,12 @@ public abstract class BaseHomePageApiTest extends AbstractControllerTest { AdminSettings twoFaSettingsSettings = new AdminSettings(); twoFaSettingsSettings.setKey("twoFaSettings"); - twoFaSettingsSettings.setJsonValue(JacksonUtil.newObjectNode()); + + var twoFaSettings = JacksonUtil.newObjectNode(); + var providers = JacksonUtil.newArrayNode(); + providers.add(JacksonUtil.newObjectNode()); + twoFaSettings.set("providers", providers); + twoFaSettingsSettings.setJsonValue(twoFaSettings); doPost("/api/admin/settings", twoFaSettingsSettings).andExpect(status().isOk()); featuresInfo = doGet("/api/admin/featuresInfo", FeaturesInfo.class); @@ -317,7 +353,13 @@ public abstract class BaseHomePageApiTest extends AbstractControllerTest { AdminSettings notificationsSettings = new AdminSettings(); notificationsSettings.setKey("notifications"); - notificationsSettings.setJsonValue(JacksonUtil.newObjectNode()); + + var notificationSettings = JacksonUtil.newObjectNode(); + var deliveryMethodsConfigs = JacksonUtil.newObjectNode(); + deliveryMethodsConfigs.set("SLACK", JacksonUtil.newObjectNode()); + notificationSettings.set("deliveryMethodsConfigs", deliveryMethodsConfigs); + + notificationsSettings.setJsonValue(notificationSettings); doPost("/api/admin/settings", notificationsSettings).andExpect(status().isOk()); featuresInfo = doGet("/api/admin/featuresInfo", FeaturesInfo.class); From 6054a14424fad68ae835ca20d9f018e4b889bd44 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 18 Apr 2023 20:16:10 +0200 Subject: [PATCH 08/10] fixed tests --- .../server/controller/BaseHomePageApiTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseHomePageApiTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseHomePageApiTest.java index a02c867948..b71be1c3b6 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseHomePageApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseHomePageApiTest.java @@ -59,7 +59,6 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.common.stats.TbApiUsageStateClient; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.service.security.auth.mfa.config.TwoFaConfigManager; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityCountCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityCountUpdate; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityDataUpdate; @@ -91,9 +90,6 @@ public abstract class BaseHomePageApiTest extends AbstractControllerTest { @MockBean private SmsService smsService; - @MockBean - TwoFaConfigManager twoFaConfigManager; - //For system administrator @Test public void testTenantsCountWsCmd() throws Exception { @@ -380,6 +376,10 @@ public abstract class BaseHomePageApiTest extends AbstractControllerTest { Assert.assertTrue(featuresInfo.isTwoFaEnabled()); Assert.assertTrue(featuresInfo.isNotificationEnabled()); Assert.assertTrue(featuresInfo.isOauthEnabled()); + + adminSettingsService.deleteAdminSettingsByTenantIdAndKey(TenantId.SYS_TENANT_ID, "notifications"); + adminSettingsService.deleteAdminSettingsByTenantIdAndKey(TenantId.SYS_TENANT_ID, "twoFaSettings"); + adminSettingsService.deleteAdminSettingsByTenantIdAndKey(TenantId.SYS_TENANT_ID, "sms"); } @Test From 46194c3bc5fc37c83f11f61cc46f8809b2cc3f0d Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 19 Apr 2023 12:42:47 +0300 Subject: [PATCH 09/10] UI: Add init value for MQTT transport configuration in device profile --- ui-ngx/src/app/shared/models/device.models.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ui-ngx/src/app/shared/models/device.models.ts b/ui-ngx/src/app/shared/models/device.models.ts index de240934c4..1cf06e9ec6 100644 --- a/ui-ngx/src/app/shared/models/device.models.ts +++ b/ui-ngx/src/app/shared/models/device.models.ts @@ -244,6 +244,7 @@ export interface DefaultDeviceProfileTransportConfiguration { export interface MqttDeviceProfileTransportConfiguration { deviceTelemetryTopic?: string; deviceAttributesTopic?: string; + deviceAttributesSubscribeTopic?: string; sparkplug?: boolean; sendAckOnValidationException?: boolean; transportPayloadTypeConfiguration?: { @@ -365,6 +366,7 @@ export function createDeviceProfileTransportConfiguration(type: DeviceTransportT const mqttTransportConfiguration: MqttDeviceProfileTransportConfiguration = { deviceTelemetryTopic: 'v1/devices/me/telemetry', deviceAttributesTopic: 'v1/devices/me/attributes', + deviceAttributesSubscribeTopic: 'v1/devices/me/attributes', sparkplug: false, sendAckOnValidationException: false, transportPayloadTypeConfiguration: { From 61466d02763fb7f332b29afb35d6f51cda8203e5 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 19 Apr 2023 12:40:51 +0200 Subject: [PATCH 10/10] added log to TenantActor for investigation --- .../java/org/thingsboard/server/actors/tenant/TenantActor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 4669907fbd..84b7757846 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -96,6 +96,7 @@ public class TenantActor extends RuleChainManagerActor { log.info("[{}] Skip init of the rule chains due to API limits", tenantId); } } catch (Exception e) { + log.info("Failed to check ApiUsage \"ReExecEnabled\"!!!", e); cantFindTenant = true; } }