From 442fe9029e41a6be4a5168b46e56942c93342c93 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 13 Nov 2023 13:14:37 +0200 Subject: [PATCH 01/17] added env variable to regulate pubsud queue executor threads pool size --- application/src/main/resources/thingsboard.yml | 2 ++ .../server/queue/pubsub/TbPubSubProducerTemplate.java | 10 +++++++++- .../server/queue/pubsub/TbPubSubSettings.java | 3 +++ msa/vc-executor/src/main/resources/tb-vc-executor.yml | 2 ++ .../coap/src/main/resources/tb-coap-transport.yml | 2 ++ .../http/src/main/resources/tb-http-transport.yml | 2 ++ .../lwm2m/src/main/resources/tb-lwm2m-transport.yml | 2 ++ .../mqtt/src/main/resources/tb-mqtt-transport.yml | 2 ++ .../snmp/src/main/resources/tb-snmp-transport.yml | 2 ++ 9 files changed, 26 insertions(+), 1 deletion(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index bdf0ad8252..e292237da5 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1411,6 +1411,8 @@ queue: max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + # Number of threads of pubsub executor provider + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:50}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java index 0d84a8f839..56e21da3db 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.pubsub; import com.google.api.core.ApiFuture; import com.google.api.core.ApiFutureCallback; import com.google.api.core.ApiFutures; +import com.google.api.gax.core.InstantiatingExecutorProvider; import com.google.cloud.pubsub.v1.Publisher; import com.google.gson.Gson; import com.google.protobuf.ByteString; @@ -120,7 +121,14 @@ public class TbPubSubProducerTemplate implements TbQueuePr try { admin.createTopicIfNotExists(topic); ProjectTopicName topicName = ProjectTopicName.of(pubSubSettings.getProjectId(), topic); - Publisher publisher = Publisher.newBuilder(topicName).setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); + InstantiatingExecutorProvider executorProvider = + InstantiatingExecutorProvider.newBuilder() + .setExecutorThreadCount(pubSubSettings.getExecutorThreadPoolSize()) + .build(); + Publisher publisher = Publisher.newBuilder(topicName) + .setCredentialsProvider(pubSubSettings.getCredentialsProvider()) + .setExecutorProvider(executorProvider) + .build(); publisherMap.put(topic, publisher); return publisher; } catch (IOException e) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java index fe9d85c306..b1b095f23c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java @@ -46,6 +46,9 @@ public class TbPubSubSettings { @Value("${queue.pubsub.max_messages}") private int maxMessages; + @Value("${queue.pubsub.executor_thread_pool_size}") + private int executorThreadPoolSize; + private CredentialsProvider credentialsProvider; @PostConstruct diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 4ffa1ab702..cfbd61eda5 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -172,6 +172,8 @@ queue: max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + # Number of threads of pubsub executor provider + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:50}" queue-properties: # Pub/Sub properties for Core subscribers, messages which will commit after ackDeadlineInSec period can be consumed again core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 437f551690..be54ed34cf 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -295,6 +295,8 @@ queue: max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + # Number of threads of pubsub executor provider + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:50}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 8d4d6fadf0..cf2c49b846 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -278,6 +278,8 @@ queue: max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" # Number of messages per a consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + # Number of threads of pubsub executor provider + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:50}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consume again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index 8fd1c11e6e..9884b33390 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -374,6 +374,8 @@ queue: max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + # Number of threads of pubsub executor provider + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:50}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 6c51d18ab7..0c26caa97e 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -311,6 +311,8 @@ queue: max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + # Number of threads of pubsub executor provider + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:50}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index eeb6807714..5e4dc06275 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -264,6 +264,8 @@ queue: max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + # Number of threads of pubsub executor provider + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:50}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" From ccb7b947009b0da2076b7faa8bf1c212cff066c7 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 13 Nov 2023 13:17:19 +0200 Subject: [PATCH 02/17] refactoring --- application/src/main/resources/thingsboard.yml | 2 +- msa/vc-executor/src/main/resources/tb-vc-executor.yml | 2 +- transport/coap/src/main/resources/tb-coap-transport.yml | 2 +- transport/http/src/main/resources/tb-http-transport.yml | 2 +- transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml | 2 +- transport/mqtt/src/main/resources/tb-mqtt-transport.yml | 2 +- transport/snmp/src/main/resources/tb-snmp-transport.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index e292237da5..5c2a62a79a 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1412,7 +1412,7 @@ queue: # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" # Number of threads of pubsub executor provider - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:50}" + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:50}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index cfbd61eda5..63ed58da12 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -173,7 +173,7 @@ queue: # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" # Number of threads of pubsub executor provider - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:50}" + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:50}" queue-properties: # Pub/Sub properties for Core subscribers, messages which will commit after ackDeadlineInSec period can be consumed again core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index be54ed34cf..e2c515de58 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -296,7 +296,7 @@ queue: # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" # Number of threads of pubsub executor provider - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:50}" + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:50}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index cf2c49b846..820c901603 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -279,7 +279,7 @@ queue: # Number of messages per a consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" # Number of threads of pubsub executor provider - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:50}" + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:50}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consume again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index 9884b33390..f7e7d55545 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -375,7 +375,7 @@ queue: # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" # Number of threads of pubsub executor provider - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:50}" + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:50}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 0c26caa97e..d67f5f91ba 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -312,7 +312,7 @@ queue: # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" # Number of threads of pubsub executor provider - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:50}" + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:50}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index 5e4dc06275..bdd9aef7aa 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -265,7 +265,7 @@ queue: # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" # Number of threads of pubsub executor provider - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:50}" + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:50}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" From ccb0fc8cad395fe176a01977469494c5a2ba476f Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 13 Nov 2023 15:29:22 +0200 Subject: [PATCH 03/17] updated TbPubSubNode publisher to use one thread for publishing --- .../thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java index c55113783c..060683434f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java @@ -20,6 +20,7 @@ import com.google.api.core.ApiFutureCallback; import com.google.api.core.ApiFutures; import com.google.api.gax.core.CredentialsProvider; import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.api.gax.core.InstantiatingExecutorProvider; import com.google.api.gax.retrying.RetrySettings; import com.google.auth.oauth2.ServiceAccountCredentials; import com.google.cloud.pubsub.v1.Publisher; @@ -135,6 +136,11 @@ public class TbPubSubNode extends TbAbstractExternalNode { new ByteArrayInputStream(config.getServiceAccountKey().getBytes())); CredentialsProvider credProvider = FixedCredentialsProvider.create(credentials); + InstantiatingExecutorProvider executorProvider = + InstantiatingExecutorProvider.newBuilder() + .setExecutorThreadCount(1) + .build(); + var retrySettings = RetrySettings.newBuilder() .setTotalTimeout(Duration.ofSeconds(10)) .setInitialRetryDelay(Duration.ofMillis(50)) @@ -148,6 +154,7 @@ public class TbPubSubNode extends TbAbstractExternalNode { return Publisher.newBuilder(topicName) .setCredentialsProvider(credProvider) .setRetrySettings(retrySettings) + .setExecutorProvider(executorProvider) .build(); } } From 4f22ef9733a38d486b525e7d08c08a978a8bd931 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Tue, 14 Nov 2023 17:36:51 +0200 Subject: [PATCH 04/17] created PubSubExecutor to use each time publisher is created and do not produce too much threads created --- .../server/actors/ActorSystemContext.java | 5 ++++ .../actors/ruleChain/DefaultTbContext.java | 6 +++++ .../executors/PubSubExecutorService.java | 26 +++++++++++++++++++ .../pubsub/TbPubSubProducerTemplate.java | 13 +++++++++- common/util/pom.xml | 4 +++ .../common/util/PubSubExecutor.java | 7 +++++ .../rule/engine/api/TbContext.java | 3 +++ .../rule/engine/gcp/pubsub/TbPubSubNode.java | 6 +++-- 8 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/executors/PubSubExecutorService.java create mode 100644 common/util/src/main/java/org/thingsboard/common/util/PubSubExecutor.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 3d4340f6c9..b9f539c877 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -29,6 +29,7 @@ import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.service.executors.PubSubExecutorService; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.NotificationCenter; import org.thingsboard.rule.engine.api.SmsService; @@ -322,6 +323,10 @@ public class ActorSystemContext { @Getter private NotificationExecutorService notificationExecutor; + @Autowired + @Getter + private PubSubExecutorService pubSubExecutorService; + @Autowired @Getter private SharedEventLoopGroupService sharedEventLoopGroupService; diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index ca1d174322..299b6274b0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -22,6 +22,7 @@ import lombok.extern.slf4j.Slf4j; import org.bouncycastle.util.Arrays; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ListeningExecutor; +import org.thingsboard.server.service.executors.PubSubExecutorService; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.NotificationCenter; import org.thingsboard.rule.engine.api.RuleEngineAlarmService; @@ -538,6 +539,11 @@ class DefaultTbContext implements TbContext { return mainCtx.getNotificationExecutor(); } + @Override + public PubSubExecutorService getPubsubExecutor() { + return mainCtx.getPubSubExecutorService(); + } + @Override @Deprecated public ScriptEngine createJsScriptEngine(String script, String... argNames) { diff --git a/application/src/main/java/org/thingsboard/server/service/executors/PubSubExecutorService.java b/application/src/main/java/org/thingsboard/server/service/executors/PubSubExecutorService.java new file mode 100644 index 0000000000..22d55cc673 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/executors/PubSubExecutorService.java @@ -0,0 +1,26 @@ +package org.thingsboard.server.service.executors; + +import com.google.api.gax.core.FixedExecutorProvider; +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.PubSubExecutor; +import org.thingsboard.common.util.ThingsBoardThreadFactory; + +import javax.annotation.PostConstruct; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + +@Component +public class PubSubExecutorService implements PubSubExecutor { + private static final int THREADS_PER_CPU = 5; + private FixedExecutorProvider fixedExecutorProvider; + + @PostConstruct + public void init() { + ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(THREADS_PER_CPU * Runtime.getRuntime().availableProcessors(), ThingsBoardThreadFactory.forName("tb-pubsub-producer-scheduler"));; + this.fixedExecutorProvider = FixedExecutorProvider.create(scheduler); + } + + public FixedExecutorProvider getExecutorProvider() { + return fixedExecutorProvider; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java index 0d84a8f839..46c6bb6413 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java @@ -18,12 +18,14 @@ package org.thingsboard.server.queue.pubsub; import com.google.api.core.ApiFuture; import com.google.api.core.ApiFutureCallback; import com.google.api.core.ApiFutures; +import com.google.api.gax.core.FixedExecutorProvider; import com.google.cloud.pubsub.v1.Publisher; import com.google.gson.Gson; import com.google.protobuf.ByteString; import com.google.pubsub.v1.ProjectTopicName; import com.google.pubsub.v1.PubsubMessage; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueCallback; @@ -36,6 +38,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @Slf4j @@ -50,11 +53,16 @@ public class TbPubSubProducerTemplate implements TbQueuePr private final Map publisherMap = new ConcurrentHashMap<>(); private final ExecutorService pubExecutor = Executors.newCachedThreadPool(); + private static final int THREADS_PER_CPU = 5; + private final FixedExecutorProvider fixedExecutorProvider; public TbPubSubProducerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String defaultTopic) { this.defaultTopic = defaultTopic; this.admin = admin; this.pubSubSettings = pubSubSettings; + + ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(THREADS_PER_CPU * Runtime.getRuntime().availableProcessors(), ThingsBoardThreadFactory.forName("tb-pubsub-producer-scheduler"));; + fixedExecutorProvider = FixedExecutorProvider.create(scheduler); } @Override @@ -120,7 +128,10 @@ public class TbPubSubProducerTemplate implements TbQueuePr try { admin.createTopicIfNotExists(topic); ProjectTopicName topicName = ProjectTopicName.of(pubSubSettings.getProjectId(), topic); - Publisher publisher = Publisher.newBuilder(topicName).setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); + Publisher publisher = Publisher.newBuilder(topicName) + .setCredentialsProvider(pubSubSettings.getCredentialsProvider()) + .setExecutorProvider(fixedExecutorProvider) + .build(); publisherMap.put(topic, publisher); return publisher; } catch (IOException e) { diff --git a/common/util/pom.xml b/common/util/pom.xml index eab8ae982b..c2f5dfea6c 100644 --- a/common/util/pom.xml +++ b/common/util/pom.xml @@ -53,6 +53,10 @@ guava provided + + com.google.cloud + google-cloud-pubsub + javax.annotation javax.annotation-api diff --git a/common/util/src/main/java/org/thingsboard/common/util/PubSubExecutor.java b/common/util/src/main/java/org/thingsboard/common/util/PubSubExecutor.java new file mode 100644 index 0000000000..5354102578 --- /dev/null +++ b/common/util/src/main/java/org/thingsboard/common/util/PubSubExecutor.java @@ -0,0 +1,7 @@ +package org.thingsboard.common.util; + +import com.google.api.gax.core.FixedExecutorProvider; + +public interface PubSubExecutor { + FixedExecutorProvider getExecutorProvider(); +} diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index e645110a9d..cd71626afd 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -17,6 +17,7 @@ package org.thingsboard.rule.engine.api; import io.netty.channel.EventLoopGroup; import org.thingsboard.common.util.ListeningExecutor; +import org.thingsboard.common.util.PubSubExecutor; import org.thingsboard.rule.engine.api.slack.SlackService; import org.thingsboard.rule.engine.api.sms.SmsSenderFactory; import org.thingsboard.server.cluster.TbClusterService; @@ -318,6 +319,8 @@ public interface TbContext { ListeningExecutor getNotificationExecutor(); + PubSubExecutor getPubsubExecutor(); + MailService getMailService(boolean isSystem); SmsService getSmsService(); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java index c55113783c..4c24bdf1cb 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java @@ -20,6 +20,7 @@ import com.google.api.core.ApiFutureCallback; import com.google.api.core.ApiFutures; import com.google.api.gax.core.CredentialsProvider; import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.api.gax.core.FixedExecutorProvider; import com.google.api.gax.retrying.RetrySettings; import com.google.auth.oauth2.ServiceAccountCredentials; import com.google.cloud.pubsub.v1.Publisher; @@ -68,7 +69,7 @@ public class TbPubSubNode extends TbAbstractExternalNode { super.init(ctx); this.config = TbNodeUtils.convert(configuration, TbPubSubNodeConfiguration.class); try { - this.pubSubClient = initPubSubClient(); + this.pubSubClient = initPubSubClient(ctx.getPubsubExecutor().getExecutorProvider()); } catch (Exception e) { throw new TbNodeException(e); } @@ -128,7 +129,7 @@ public class TbPubSubNode extends TbAbstractExternalNode { return TbMsg.transformMsgMetadata(origMsg, metaData); } - private Publisher initPubSubClient() throws IOException { + private Publisher initPubSubClient(FixedExecutorProvider fixedExecutorProvider) throws IOException { ProjectTopicName topicName = ProjectTopicName.of(config.getProjectId(), config.getTopicName()); ServiceAccountCredentials credentials = ServiceAccountCredentials.fromStream( @@ -148,6 +149,7 @@ public class TbPubSubNode extends TbAbstractExternalNode { return Publisher.newBuilder(topicName) .setCredentialsProvider(credProvider) .setRetrySettings(retrySettings) + .setExecutorProvider(fixedExecutorProvider) .build(); } } From 08727b4c368b676c0b62e2dd136ef83fec14381f Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 15 Nov 2023 14:51:58 +0200 Subject: [PATCH 05/17] added executor provider for pubsub queue, added env variable to manage pubsub queue executor provider thread pool --- .../server/actors/ActorSystemContext.java | 4 +- .../actors/ruleChain/DefaultTbContext.java | 6 +-- .../executors/PubSubExecutorService.java | 26 ---------- .../PubSubRuleNodeExecutorProvider.java | 29 +++++++++++ .../src/main/resources/thingsboard.yml | 8 +-- .../provider/PubSubMonolithQueueFactory.java | 50 +++++++++++-------- .../provider/PubSubTbCoreQueueFactory.java | 38 ++++++++------ .../PubSubTbRuleEngineQueueFactory.java | 28 ++++++----- .../PubSubTbVersionControlQueueFactory.java | 13 +++-- .../provider/PubSubTransportQueueFactory.java | 20 +++++--- .../pubsub/TbPubSubConsumerTemplate.java | 4 +- .../pubsub/TbPubSubProducerTemplate.java | 7 ++- .../pubsub/TbPubSubQueueExecutorProvider.java | 37 ++++++++++++++ common/util/pom.xml | 4 -- .../common/util/ExecutorProvider.java | 8 +++ .../common/util/PubSubExecutor.java | 7 --- .../src/main/resources/tb-vc-executor.yml | 2 + .../rule/engine/api/TbContext.java | 4 +- .../rule/engine/gcp/pubsub/TbPubSubNode.java | 2 +- .../src/main/resources/tb-coap-transport.yml | 2 + .../src/main/resources/tb-http-transport.yml | 2 + .../src/main/resources/tb-lwm2m-transport.yml | 2 + .../src/main/resources/tb-mqtt-transport.yml | 2 + .../src/main/resources/tb-snmp-transport.yml | 2 + 24 files changed, 194 insertions(+), 113 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/executors/PubSubExecutorService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/executors/PubSubRuleNodeExecutorProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubQueueExecutorProvider.java create mode 100644 common/util/src/main/java/org/thingsboard/common/util/ExecutorProvider.java delete mode 100644 common/util/src/main/java/org/thingsboard/common/util/PubSubExecutor.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index b9f539c877..24a240661f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -29,7 +29,7 @@ import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.service.executors.PubSubExecutorService; +import org.thingsboard.server.service.executors.PubSubRuleNodeExecutorProvider; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.NotificationCenter; import org.thingsboard.rule.engine.api.SmsService; @@ -325,7 +325,7 @@ public class ActorSystemContext { @Autowired @Getter - private PubSubExecutorService pubSubExecutorService; + private PubSubRuleNodeExecutorProvider pubSubRuleNodeExecutorProvider; @Autowired @Getter diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 299b6274b0..28963b383a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -22,7 +22,6 @@ import lombok.extern.slf4j.Slf4j; import org.bouncycastle.util.Arrays; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.server.service.executors.PubSubExecutorService; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.NotificationCenter; import org.thingsboard.rule.engine.api.RuleEngineAlarmService; @@ -105,6 +104,7 @@ import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.service.executors.PubSubRuleNodeExecutorProvider; import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import org.thingsboard.server.service.script.RuleNodeTbelScriptEngine; @@ -540,8 +540,8 @@ class DefaultTbContext implements TbContext { } @Override - public PubSubExecutorService getPubsubExecutor() { - return mainCtx.getPubSubExecutorService(); + public PubSubRuleNodeExecutorProvider getPubSubRuleNodeExecutorProvider() { + return mainCtx.getPubSubRuleNodeExecutorProvider(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/executors/PubSubExecutorService.java b/application/src/main/java/org/thingsboard/server/service/executors/PubSubExecutorService.java deleted file mode 100644 index 22d55cc673..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/executors/PubSubExecutorService.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.thingsboard.server.service.executors; - -import com.google.api.gax.core.FixedExecutorProvider; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.PubSubExecutor; -import org.thingsboard.common.util.ThingsBoardThreadFactory; - -import javax.annotation.PostConstruct; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; - -@Component -public class PubSubExecutorService implements PubSubExecutor { - private static final int THREADS_PER_CPU = 5; - private FixedExecutorProvider fixedExecutorProvider; - - @PostConstruct - public void init() { - ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(THREADS_PER_CPU * Runtime.getRuntime().availableProcessors(), ThingsBoardThreadFactory.forName("tb-pubsub-producer-scheduler"));; - this.fixedExecutorProvider = FixedExecutorProvider.create(scheduler); - } - - public FixedExecutorProvider getExecutorProvider() { - return fixedExecutorProvider; - } -} diff --git a/application/src/main/java/org/thingsboard/server/service/executors/PubSubRuleNodeExecutorProvider.java b/application/src/main/java/org/thingsboard/server/service/executors/PubSubRuleNodeExecutorProvider.java new file mode 100644 index 0000000000..7db74f6195 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/executors/PubSubRuleNodeExecutorProvider.java @@ -0,0 +1,29 @@ +package org.thingsboard.server.service.executors; + +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.ExecutorProvider; +import org.thingsboard.common.util.ThingsBoardThreadFactory; + +import javax.annotation.PostConstruct; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + +@Component +public class PubSubRuleNodeExecutorProvider implements ExecutorProvider { + + /** + * Refers to com.google.cloud.pubsub.v1.Publisher default executor configuration + */ + private static final int THREADS_PER_CPU = 5; + private ScheduledExecutorService executor; + + @PostConstruct + public void init() { + executor = Executors.newScheduledThreadPool(THREADS_PER_CPU * Runtime.getRuntime().availableProcessors(), ThingsBoardThreadFactory.forName("pubsub-rule-nodes"));; + } + + @Override + public ScheduledExecutorService getExecutor() { + return executor; + } +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index bdf0ad8252..114dafa6f0 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1270,7 +1270,7 @@ swagger: # Queue configuration parameters queue: - type: "${TB_QUEUE_TYPE:in-memory}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) + type: "${TB_QUEUE_TYPE:pubsub}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka) except of js executor topics (please use REMOTE_JS_EVAL_REQUEST_TOPIC and REMOTE_JS_EVAL_RESPONSE_TOPIC to specify custom topic names) in_memory: stats: @@ -1404,13 +1404,15 @@ queue: version-control: "${TB_QUEUE_AWS_SQS_VC_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" pubsub: # Project ID from Google Cloud - project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:pubsub-389210}" # API Credentials in JSON format - service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:{\n \"type\": \"service_account\",\n \"project_id\": \"pubsub-389210\",\n \"private_key_id\": \"e83e233f60bb75f52e6ee9d34046c1e7b5e3f2e6\",\n \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCmram5Oa60kyFM\\nFwo7d4yYA2U4vLUa2HV7h5G5ZxhXTPk+V4eYneBPSX0bJ/V5L4z7N/xNNwDR80i3\\n+tCFIf9P+jS3sE0ThZDc8Rdgo1Ln8nm4qf/9uBrOdSjn5VKKwvnNXphwTniwzssS\\nWn3AMuL8flgKoOiRaBOMXykIWN4/epKmv1tmnnq57XLW2cRTsE+pCcbqn8W2LEgA\\nNP+NJvvhQ6wsU341F2oTMAsuwsmiOZ1aiKKCafjGb/STPYy/WXsLjrfwK08U2SJh\\n+hHyvzdH1AU40khelnLjQhrltUGk1VAaKtucDKGSAoL5JIkH0Q5qczz1Ys6l3VvK\\nlhDNP8NHAgMBAAECggEAFUVJSRGC6ZlzIN452I/vmcCpLL3sUMI57+tDngj5kr6I\\neju6WrAfHY8vz4TgbzgxwieJY6M20BQ/ffccoDjP97li++QcWfbsHz4HMTZ2kJIh\\nlX7gY9UYWquZ7koWKA4sydgeFQr2nP2u7actsBbuX7GR871IILJK+Fl2h5Grvybk\\nz70RzLY+M26ZVBf+ejKlyaoMVLQ6w272llc5/SJ4jmsbn1urLM2+ChgAjOeqo7Qp\\nBBjUJmjC9b5Tev0jg+FXdw+ba41bXmYM5x265buwbY4vYyIJ+XF6dEl36cdVa1xI\\n+XISQZ66SuvEkHRc+x1hEEMymOtt8d++SeFn4/BkDQKBgQDkGtYA/8751C8xDB4+\\nlpxXf3wN/QjWPpXAV1fyj9sgwz05M1E0KzYpX6H2rWVqFhPkuqZUCeZ5Iqjvybp7\\n8upLJ+lRpHag1dkS4ElICNNcHhIQB9xgmFtdH4euTzuAFTd5nZ43oRt6gGEo0I/S\\nmKmupS2IwpM2/Qm6dP7l2ruS5QKBgQC7D8eTIIu8x2plJ6rebEAneX6aAUzQ9chk\\nRB53wjDx0UCXC/J2l1PX0vGGY8JOQdmDS6MPdFaVZxnqtrrGNBDuSArDNH7bV0ab\\nBKnLStPOwkehQEhg6ShHX7r4EPs6GBTWDQnnCgR8KGq+7rdo/lz7C/TK8+xqRaWq\\nYCUMY4g+uwKBgHtPFokLwHPFhI1bI65p9LJINGPLec93nbSQgvaZVbfsU1hsqWUu\\nRuUu0XtsWPp0XOS1Ed00TOcHGZQm1SzUMFvYg4SjB44CjIprvLG4M6oEh3crCLMf\\nKaS5urs8Eco3rfmMf09LRHOAmwVZWaZa6L+Eg4z+wl5jg7LMNE9FY0kRAoGACU+J\\nwXr3OZg7ZXmJ+bQtpUlY0dWKu0Pgi40QbymNQhwGOP5xPRHfLHjlaKkCfN6uMujK\\n3vQNczZEhfg+Z6sjxJh1YK32OninnQOoZ+P7kuj8o7wNXjV8ucC8D6jYuFWGg/j2\\nKzfGbV+doI9FNcajXiOENa3acJey3T4X2fwCRg0CgYEA0m3iO8HvnWhAs+clP2Ph\\nI8mXNbNbtCa7bOPmhdidSwJpbXNNwKNU7xAXHHueFGlEnNFsQrqy7bPfol4ABOhl\\nV3Zdsuc5v1OXM7IrLZ16cC7fogydSQz8QK86TcC50u/aSmkaevujsJfabC/YF7sR\\nBl8tPN7I4H72JKOuUTn/Sfo=\\n-----END PRIVATE KEY-----\\n\",\n \"client_email\": \"pubsubaccountname@pubsub-389210.iam.gserviceaccount.com\",\n \"client_id\": \"103395736410565244836\",\n \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\n \"token_uri\": \"https://oauth2.googleapis.com/token\",\n \"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\n \"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/pubsubaccountname%40pubsub-389210.iam.gserviceaccount.com\",\n \"universe_domain\": \"googleapis.com\"\n}}" # Message size for PubSub queue.Value in bytes max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + # Number of threads of pubsub executor provider + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index 554d452556..8ecd6564c8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -44,6 +44,7 @@ import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubQueueExecutorProvider; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; @@ -77,6 +78,7 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin vcAdmin; + private final TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider; public PubSubMonolithQueueFactory(TbPubSubSettings pubSubSettings, TbQueueCoreSettings coreSettings, @@ -87,7 +89,8 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng TbServiceInfoProvider serviceInfoProvider, TbPubSubSubscriptionSettings pubSubSubscriptionSettings, TbQueueRemoteJsInvokeSettings jsInvokeSettings, - TbQueueVersionControlSettings vcSettings) { + TbQueueVersionControlSettings vcSettings, + TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; @@ -96,6 +99,7 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng this.topicService = topicService; this.serviceInfoProvider = serviceInfoProvider; this.vcSettings = vcSettings; + this.tbPubSubQueueExecutorProvider = tbPubSubQueueExecutorProvider; this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); @@ -109,85 +113,91 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, transportNotificationSettings.getNotificationsTopic()); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, transportNotificationSettings.getNotificationsTopic(), tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic(), tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, ruleEngineSettings.getTopic()); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, ruleEngineSettings.getTopic(), tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, coreSettings.getTopic(), tbPubSubQueueExecutorProvider); } @Override public TbQueueConsumer> createToVersionControlMsgConsumer() { return new TbPubSubConsumerTemplate<>(vcAdmin, pubSubSettings, vcSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()), + tbPubSubQueueExecutorProvider ); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, configuration.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()), + tbPubSubQueueExecutorProvider); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, topicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()), + tbPubSubQueueExecutorProvider); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()), + tbPubSubQueueExecutorProvider); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, topicService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()), + tbPubSubQueueExecutorProvider); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders()), + tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getResponsesTopic()); + return new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getResponsesTopic(), tbPubSubQueueExecutorProvider); } @Override @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic()); + TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic(), tbPubSubQueueExecutorProvider); TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), msg -> { RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder(); JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); + }, tbPubSubQueueExecutorProvider); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); @@ -203,28 +213,28 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng @Override public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()), tbPubSubQueueExecutorProvider); } @Override public TbQueueConsumer> createToOtaPackageStateServiceMsgConsumer() { return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getOtaPackageTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToOtaPackageStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToOtaPackageStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders()), tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getOtaPackageTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getOtaPackageTopic(), tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic(), tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createVersionControlMsgProducer() { - return new TbPubSubProducerTemplate<>(vcAdmin, pubSubSettings, vcSettings.getTopic()); + return new TbPubSubProducerTemplate<>(vcAdmin, pubSubSettings, vcSettings.getTopic(), tbPubSubQueueExecutorProvider); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 3b45992e25..5443339d2a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -42,6 +42,7 @@ import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubQueueExecutorProvider; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; @@ -72,6 +73,7 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin ruleEngineAdmin; + private final TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider; public PubSubTbCoreQueueFactory(TbPubSubSettings pubSubSettings, TbQueueCoreSettings coreSettings, @@ -81,7 +83,8 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbQueueRuleEngineSettings ruleEngineSettings, - TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { + TbPubSubSubscriptionSettings pubSubSubscriptionSettings, + TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.transportApiSettings = transportApiSettings; @@ -90,6 +93,7 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { this.jsInvokeSettings = jsInvokeSettings; this.transportNotificationSettings = transportNotificationSettings; this.ruleEngineSettings = ruleEngineSettings; + this.tbPubSubQueueExecutorProvider = tbPubSubQueueExecutorProvider; this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); @@ -100,64 +104,64 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, transportNotificationSettings.getNotificationsTopic()); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, transportNotificationSettings.getNotificationsTopic(), tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, ruleEngineSettings.getTopic()); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, ruleEngineSettings.getTopic(), tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, coreSettings.getTopic()); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, coreSettings.getTopic(), tbPubSubQueueExecutorProvider); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()), tbPubSubQueueExecutorProvider); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, topicService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()), tbPubSubQueueExecutorProvider); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders()), tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getResponsesTopic()); + return new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getResponsesTopic(), tbPubSubQueueExecutorProvider); } @Override @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic()); + TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic(), tbPubSubQueueExecutorProvider); TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), msg -> { JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); + }, tbPubSubQueueExecutorProvider); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); @@ -173,23 +177,25 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()), + tbPubSubQueueExecutorProvider); } @Override public TbQueueConsumer> createToOtaPackageStateServiceMsgConsumer() { return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getOtaPackageTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToOtaPackageStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToOtaPackageStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders()), + tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getOtaPackageTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getOtaPackageTopic(), tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic(), tbPubSubQueueExecutorProvider); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java index a60f77852f..b0fde82e7d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -40,6 +40,7 @@ import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubQueueExecutorProvider; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; @@ -62,6 +63,7 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -75,7 +77,8 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory TbServiceInfoProvider serviceInfoProvider, TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbQueueTransportNotificationSettings transportNotificationSettings, - TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { + TbPubSubSubscriptionSettings pubSubSubscriptionSettings, + TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; @@ -83,6 +86,7 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory this.serviceInfoProvider = serviceInfoProvider; this.jsInvokeSettings = jsInvokeSettings; this.transportNotificationSettings = transportNotificationSettings; + this.tbPubSubQueueExecutorProvider = tbPubSubQueueExecutorProvider; this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); @@ -92,53 +96,53 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic())); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic()), tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, topicService.buildTopicName(ruleEngineSettings.getTopic()), tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(ruleEngineSettings.getTopic()), tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic())); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic()), tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic())); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic()), tbPubSubQueueExecutorProvider); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, topicService.buildTopicName(configuration.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()), tbPubSubQueueExecutorProvider); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, topicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()), tbPubSubQueueExecutorProvider); } @Override @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic()); + TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic(), tbPubSubQueueExecutorProvider); TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), msg -> { JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); + }, tbPubSubQueueExecutorProvider); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); @@ -153,12 +157,12 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory @Override public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic()), tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic()), tbPubSubQueueExecutorProvider); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java index 8154d04d6d..29207ed429 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubQueueExecutorProvider; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; @@ -41,6 +42,7 @@ public class PubSubTbVersionControlQueueFactory implements TbVersionControlQueue private final TbQueueCoreSettings coreSettings; private final TbQueueVersionControlSettings vcSettings; private final TopicService topicService; + private final TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin notificationAdmin; @@ -50,12 +52,14 @@ public class PubSubTbVersionControlQueueFactory implements TbVersionControlQueue TbQueueCoreSettings coreSettings, TbQueueVersionControlSettings vcSettings, TbPubSubSubscriptionSettings pubSubSubscriptionSettings, - TopicService topicService + TopicService topicService, + TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider ) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.vcSettings = vcSettings; this.topicService = topicService; + this.tbPubSubQueueExecutorProvider = tbPubSubQueueExecutorProvider; this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); @@ -64,18 +68,19 @@ public class PubSubTbVersionControlQueueFactory implements TbVersionControlQueue @Override public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic()), tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic())); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic()), tbPubSubQueueExecutorProvider); } @Override public TbQueueConsumer> createToVersionControlMsgConsumer() { return new TbPubSubConsumerTemplate<>(vcAdmin, pubSubSettings, topicService.buildTopicName(vcSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()), + tbPubSubQueueExecutorProvider ); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java index edcad873c4..94a454dd5b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java @@ -37,6 +37,7 @@ import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubQueueExecutorProvider; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; @@ -58,6 +59,7 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TopicService topicService; + private final TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -71,7 +73,8 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbPubSubSubscriptionSettings pubSubSubscriptionSettings, - TopicService topicService) { + TopicService topicService, + TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider) { this.pubSubSettings = pubSubSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; @@ -79,6 +82,7 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.topicService = topicService; + this.tbPubSubQueueExecutorProvider = tbPubSubQueueExecutorProvider; this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); @@ -88,10 +92,10 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbQueueProducer> producer = new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, topicService.buildTopicName(transportApiSettings.getRequestsTopic())); + TbQueueProducer> producer = new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, topicService.buildTopicName(transportApiSettings.getRequestsTopic()), tbPubSubQueueExecutorProvider); TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, topicService.buildTopicName(transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders()), tbPubSubQueueExecutorProvider); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); @@ -106,29 +110,29 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, topicService.buildTopicName(ruleEngineSettings.getTopic()), tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic())); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic()), tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic())); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic()), tbPubSubQueueExecutorProvider); } @Override public TbQueueConsumer> createTransportNotificationsConsumer() { return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders()), tbPubSubQueueExecutorProvider); } @Override public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic()), tbPubSubQueueExecutorProvider); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java index 97b7b47843..fd01696873 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.pubsub; import com.google.api.core.ApiFuture; import com.google.api.core.ApiFutures; +import com.google.api.gax.core.FixedExecutorProvider; import com.google.cloud.pubsub.v1.stub.GrpcSubscriberStub; import com.google.cloud.pubsub.v1.stub.SubscriberStub; import com.google.cloud.pubsub.v1.stub.SubscriberStubSettings; @@ -62,7 +63,7 @@ public class TbPubSubConsumerTemplate extends AbstractPara private final SubscriberStub subscriber; private volatile int messagesPerTopic; - public TbPubSubConsumerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String topic, TbQueueMsgDecoder decoder) { + public TbPubSubConsumerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String topic, TbQueueMsgDecoder decoder, TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider) { super(topic); this.admin = admin; this.pubSubSettings = pubSubSettings; @@ -76,6 +77,7 @@ public class TbPubSubConsumerTemplate extends AbstractPara SubscriberStubSettings.defaultGrpcTransportProviderBuilder() .setMaxInboundMessageSize(pubSubSettings.getMaxMsgSize()) .build()) + .setExecutorProvider(FixedExecutorProvider.create(tbPubSubQueueExecutorProvider.getExecutor())) .build(); this.subscriber = GrpcSubscriberStub.create(subscriberStubSettings); } catch (IOException e) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java index 46c6bb6413..48a104f20a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java @@ -25,6 +25,7 @@ import com.google.protobuf.ByteString; import com.google.pubsub.v1.ProjectTopicName; import com.google.pubsub.v1.PubsubMessage; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.ExecutorProvider; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; @@ -53,16 +54,14 @@ public class TbPubSubProducerTemplate implements TbQueuePr private final Map publisherMap = new ConcurrentHashMap<>(); private final ExecutorService pubExecutor = Executors.newCachedThreadPool(); - private static final int THREADS_PER_CPU = 5; private final FixedExecutorProvider fixedExecutorProvider; - public TbPubSubProducerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String defaultTopic) { + public TbPubSubProducerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String defaultTopic, ExecutorProvider executorProvider) { this.defaultTopic = defaultTopic; this.admin = admin; this.pubSubSettings = pubSubSettings; - ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(THREADS_PER_CPU * Runtime.getRuntime().availableProcessors(), ThingsBoardThreadFactory.forName("tb-pubsub-producer-scheduler"));; - fixedExecutorProvider = FixedExecutorProvider.create(scheduler); + fixedExecutorProvider = FixedExecutorProvider.create(executorProvider.getExecutor()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubQueueExecutorProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubQueueExecutorProvider.java new file mode 100644 index 0000000000..2b636b5ee2 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubQueueExecutorProvider.java @@ -0,0 +1,37 @@ +package org.thingsboard.server.queue.pubsub; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.ExecutorProvider; +import org.thingsboard.common.util.ThingsBoardThreadFactory; + +import javax.annotation.PostConstruct; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + +@ConditionalOnExpression("'${queue.type:null}'=='pubsub'") +@Component +public class TbPubSubQueueExecutorProvider implements ExecutorProvider { + + @Value("${queue.pubsub.executor_thread_pool_size}") + private Integer threadPoolSize; + /** + * Refers to com.google.cloud.pubsub.v1.Publisher default executor configuration + */ + private static final int THREADS_PER_CPU = 5; + private ScheduledExecutorService executor; + + @PostConstruct + public void init() { + if (threadPoolSize == null) { + threadPoolSize = THREADS_PER_CPU * Runtime.getRuntime().availableProcessors(); + } + executor = Executors.newScheduledThreadPool(threadPoolSize, ThingsBoardThreadFactory.forName("pubsub-queue-executor"));; + } + + @Override + public ScheduledExecutorService getExecutor() { + return executor; + } +} diff --git a/common/util/pom.xml b/common/util/pom.xml index c2f5dfea6c..eab8ae982b 100644 --- a/common/util/pom.xml +++ b/common/util/pom.xml @@ -53,10 +53,6 @@ guava provided - - com.google.cloud - google-cloud-pubsub - javax.annotation javax.annotation-api diff --git a/common/util/src/main/java/org/thingsboard/common/util/ExecutorProvider.java b/common/util/src/main/java/org/thingsboard/common/util/ExecutorProvider.java new file mode 100644 index 0000000000..4abc46fb16 --- /dev/null +++ b/common/util/src/main/java/org/thingsboard/common/util/ExecutorProvider.java @@ -0,0 +1,8 @@ +package org.thingsboard.common.util; + +import java.util.concurrent.ScheduledExecutorService; + +public interface ExecutorProvider { + + ScheduledExecutorService getExecutor(); +} diff --git a/common/util/src/main/java/org/thingsboard/common/util/PubSubExecutor.java b/common/util/src/main/java/org/thingsboard/common/util/PubSubExecutor.java deleted file mode 100644 index 5354102578..0000000000 --- a/common/util/src/main/java/org/thingsboard/common/util/PubSubExecutor.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.thingsboard.common.util; - -import com.google.api.gax.core.FixedExecutorProvider; - -public interface PubSubExecutor { - FixedExecutorProvider getExecutorProvider(); -} diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 4ffa1ab702..2353199c2b 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -172,6 +172,8 @@ queue: max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + # Number of threads of pubsub executor provider + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:}" queue-properties: # Pub/Sub properties for Core subscribers, messages which will commit after ackDeadlineInSec period can be consumed again core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index cd71626afd..40ed2c378a 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -16,8 +16,8 @@ package org.thingsboard.rule.engine.api; import io.netty.channel.EventLoopGroup; +import org.thingsboard.common.util.ExecutorProvider; import org.thingsboard.common.util.ListeningExecutor; -import org.thingsboard.common.util.PubSubExecutor; import org.thingsboard.rule.engine.api.slack.SlackService; import org.thingsboard.rule.engine.api.sms.SmsSenderFactory; import org.thingsboard.server.cluster.TbClusterService; @@ -319,7 +319,7 @@ public interface TbContext { ListeningExecutor getNotificationExecutor(); - PubSubExecutor getPubsubExecutor(); + ExecutorProvider getPubSubRuleNodeExecutorProvider(); MailService getMailService(boolean isSystem); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java index 4c24bdf1cb..39ab7bf525 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java @@ -69,7 +69,7 @@ public class TbPubSubNode extends TbAbstractExternalNode { super.init(ctx); this.config = TbNodeUtils.convert(configuration, TbPubSubNodeConfiguration.class); try { - this.pubSubClient = initPubSubClient(ctx.getPubsubExecutor().getExecutorProvider()); + this.pubSubClient = initPubSubClient(FixedExecutorProvider.create(ctx.getPubSubRuleNodeExecutorProvider().getExecutor())); } catch (Exception e) { throw new TbNodeException(e); } diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 437f551690..5588384efd 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -295,6 +295,8 @@ queue: max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + # Number of threads of pubsub executor provider + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 8d4d6fadf0..2ca2a04d1f 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -278,6 +278,8 @@ queue: max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" # Number of messages per a consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + # Number of threads of pubsub executor provider + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consume again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index 8fd1c11e6e..d8ee8d14c4 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -374,6 +374,8 @@ queue: max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + # Number of threads of pubsub executor provider + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 6c51d18ab7..6dc1213884 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -311,6 +311,8 @@ queue: max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + # Number of threads of pubsub executor provider + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index eeb6807714..928c3c0642 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -264,6 +264,8 @@ queue: max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + # Number of threads of pubsub executor provider + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" From aab09a680406b39265bb8ab9fcd3fc933fa6b55d Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 15 Nov 2023 14:56:17 +0200 Subject: [PATCH 06/17] reverted thingsboard.yml local changes --- application/src/main/resources/thingsboard.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 114dafa6f0..6961ca0655 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1404,9 +1404,9 @@ queue: version-control: "${TB_QUEUE_AWS_SQS_VC_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" pubsub: # Project ID from Google Cloud - project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:pubsub-389210}" + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" # API Credentials in JSON format - service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:{\n \"type\": \"service_account\",\n \"project_id\": \"pubsub-389210\",\n \"private_key_id\": \"e83e233f60bb75f52e6ee9d34046c1e7b5e3f2e6\",\n \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCmram5Oa60kyFM\\nFwo7d4yYA2U4vLUa2HV7h5G5ZxhXTPk+V4eYneBPSX0bJ/V5L4z7N/xNNwDR80i3\\n+tCFIf9P+jS3sE0ThZDc8Rdgo1Ln8nm4qf/9uBrOdSjn5VKKwvnNXphwTniwzssS\\nWn3AMuL8flgKoOiRaBOMXykIWN4/epKmv1tmnnq57XLW2cRTsE+pCcbqn8W2LEgA\\nNP+NJvvhQ6wsU341F2oTMAsuwsmiOZ1aiKKCafjGb/STPYy/WXsLjrfwK08U2SJh\\n+hHyvzdH1AU40khelnLjQhrltUGk1VAaKtucDKGSAoL5JIkH0Q5qczz1Ys6l3VvK\\nlhDNP8NHAgMBAAECggEAFUVJSRGC6ZlzIN452I/vmcCpLL3sUMI57+tDngj5kr6I\\neju6WrAfHY8vz4TgbzgxwieJY6M20BQ/ffccoDjP97li++QcWfbsHz4HMTZ2kJIh\\nlX7gY9UYWquZ7koWKA4sydgeFQr2nP2u7actsBbuX7GR871IILJK+Fl2h5Grvybk\\nz70RzLY+M26ZVBf+ejKlyaoMVLQ6w272llc5/SJ4jmsbn1urLM2+ChgAjOeqo7Qp\\nBBjUJmjC9b5Tev0jg+FXdw+ba41bXmYM5x265buwbY4vYyIJ+XF6dEl36cdVa1xI\\n+XISQZ66SuvEkHRc+x1hEEMymOtt8d++SeFn4/BkDQKBgQDkGtYA/8751C8xDB4+\\nlpxXf3wN/QjWPpXAV1fyj9sgwz05M1E0KzYpX6H2rWVqFhPkuqZUCeZ5Iqjvybp7\\n8upLJ+lRpHag1dkS4ElICNNcHhIQB9xgmFtdH4euTzuAFTd5nZ43oRt6gGEo0I/S\\nmKmupS2IwpM2/Qm6dP7l2ruS5QKBgQC7D8eTIIu8x2plJ6rebEAneX6aAUzQ9chk\\nRB53wjDx0UCXC/J2l1PX0vGGY8JOQdmDS6MPdFaVZxnqtrrGNBDuSArDNH7bV0ab\\nBKnLStPOwkehQEhg6ShHX7r4EPs6GBTWDQnnCgR8KGq+7rdo/lz7C/TK8+xqRaWq\\nYCUMY4g+uwKBgHtPFokLwHPFhI1bI65p9LJINGPLec93nbSQgvaZVbfsU1hsqWUu\\nRuUu0XtsWPp0XOS1Ed00TOcHGZQm1SzUMFvYg4SjB44CjIprvLG4M6oEh3crCLMf\\nKaS5urs8Eco3rfmMf09LRHOAmwVZWaZa6L+Eg4z+wl5jg7LMNE9FY0kRAoGACU+J\\nwXr3OZg7ZXmJ+bQtpUlY0dWKu0Pgi40QbymNQhwGOP5xPRHfLHjlaKkCfN6uMujK\\n3vQNczZEhfg+Z6sjxJh1YK32OninnQOoZ+P7kuj8o7wNXjV8ucC8D6jYuFWGg/j2\\nKzfGbV+doI9FNcajXiOENa3acJey3T4X2fwCRg0CgYEA0m3iO8HvnWhAs+clP2Ph\\nI8mXNbNbtCa7bOPmhdidSwJpbXNNwKNU7xAXHHueFGlEnNFsQrqy7bPfol4ABOhl\\nV3Zdsuc5v1OXM7IrLZ16cC7fogydSQz8QK86TcC50u/aSmkaevujsJfabC/YF7sR\\nBl8tPN7I4H72JKOuUTn/Sfo=\\n-----END PRIVATE KEY-----\\n\",\n \"client_email\": \"pubsubaccountname@pubsub-389210.iam.gserviceaccount.com\",\n \"client_id\": \"103395736410565244836\",\n \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\n \"token_uri\": \"https://oauth2.googleapis.com/token\",\n \"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\n \"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/pubsubaccountname%40pubsub-389210.iam.gserviceaccount.com\",\n \"universe_domain\": \"googleapis.com\"\n}}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" # Message size for PubSub queue.Value in bytes max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" # Number of messages per consumer From f26d827ab8daf5d7bf6766d0e830d7bc9831c196 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 15 Nov 2023 14:57:41 +0200 Subject: [PATCH 07/17] reverted thingsboard.yml local changes --- application/src/main/resources/thingsboard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 6961ca0655..ccc4953793 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1270,7 +1270,7 @@ swagger: # Queue configuration parameters queue: - type: "${TB_QUEUE_TYPE:pubsub}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) + type: "${TB_QUEUE_TYPE:in-memory}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka) except of js executor topics (please use REMOTE_JS_EVAL_REQUEST_TOPIC and REMOTE_JS_EVAL_RESPONSE_TOPIC to specify custom topic names) in_memory: stats: From 1be39aa9aa74c574432cd4bc5980a837f578ecf7 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 16 Nov 2023 16:16:38 +0200 Subject: [PATCH 08/17] fixed license, env name refactoring --- .../PubSubRuleNodeExecutorProvider.java | 23 +++++++++++++++++++ .../src/main/resources/thingsboard.yml | 2 +- .../pubsub/TbPubSubQueueExecutorProvider.java | 23 +++++++++++++++++++ .../common/util/ExecutorProvider.java | 15 ++++++++++++ .../src/main/resources/tb-vc-executor.yml | 2 +- .../src/main/resources/tb-coap-transport.yml | 2 +- .../src/main/resources/tb-http-transport.yml | 2 +- .../src/main/resources/tb-lwm2m-transport.yml | 2 +- .../src/main/resources/tb-mqtt-transport.yml | 2 +- .../src/main/resources/tb-snmp-transport.yml | 2 +- 10 files changed, 68 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/executors/PubSubRuleNodeExecutorProvider.java b/application/src/main/java/org/thingsboard/server/service/executors/PubSubRuleNodeExecutorProvider.java index 7db74f6195..f730949f45 100644 --- a/application/src/main/java/org/thingsboard/server/service/executors/PubSubRuleNodeExecutorProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/executors/PubSubRuleNodeExecutorProvider.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.executors; import org.springframework.stereotype.Component; @@ -5,6 +20,7 @@ import org.thingsboard.common.util.ExecutorProvider; import org.thingsboard.common.util.ThingsBoardThreadFactory; import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -26,4 +42,11 @@ public class PubSubRuleNodeExecutorProvider implements ExecutorProvider { public ScheduledExecutorService getExecutor() { return executor; } + + @PreDestroy + private void destroy() { + if (executor != null) { + executor.shutdown(); + } + } } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index ccc4953793..802756c105 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1412,7 +1412,7 @@ queue: # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" # Number of threads of pubsub executor provider - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:}" + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubQueueExecutorProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubQueueExecutorProvider.java index 2b636b5ee2..4de75faa56 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubQueueExecutorProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubQueueExecutorProvider.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.queue.pubsub; import org.springframework.beans.factory.annotation.Value; @@ -7,6 +22,7 @@ import org.thingsboard.common.util.ExecutorProvider; import org.thingsboard.common.util.ThingsBoardThreadFactory; import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -34,4 +50,11 @@ public class TbPubSubQueueExecutorProvider implements ExecutorProvider { public ScheduledExecutorService getExecutor() { return executor; } + + @PreDestroy + private void destroy() { + if (executor != null) { + executor.shutdown(); + } + } } diff --git a/common/util/src/main/java/org/thingsboard/common/util/ExecutorProvider.java b/common/util/src/main/java/org/thingsboard/common/util/ExecutorProvider.java index 4abc46fb16..61379c8c89 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/ExecutorProvider.java +++ b/common/util/src/main/java/org/thingsboard/common/util/ExecutorProvider.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.common.util; import java.util.concurrent.ScheduledExecutorService; diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 2353199c2b..9fc9b884f0 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -173,7 +173,7 @@ queue: # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" # Number of threads of pubsub executor provider - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:}" + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" queue-properties: # Pub/Sub properties for Core subscribers, messages which will commit after ackDeadlineInSec period can be consumed again core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 5588384efd..27e5c8b273 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -296,7 +296,7 @@ queue: # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" # Number of threads of pubsub executor provider - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:}" + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 2ca2a04d1f..b8070f6575 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -279,7 +279,7 @@ queue: # Number of messages per a consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" # Number of threads of pubsub executor provider - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:}" + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consume again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index d8ee8d14c4..b1d40c720f 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -375,7 +375,7 @@ queue: # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" # Number of threads of pubsub executor provider - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:}" + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 6dc1213884..4d130eecd9 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -312,7 +312,7 @@ queue: # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" # Number of threads of pubsub executor provider - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:}" + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index 928c3c0642..2b89065a55 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -265,7 +265,7 @@ queue: # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" # Number of threads of pubsub executor provider - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_SIZE:}" + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" From fbec7aab25b670b89a97b72fa252fa625247f039 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 17 Nov 2023 12:18:03 +0200 Subject: [PATCH 09/17] moved pubsub executors to pubsubSettings --- .../PubSubRuleNodeExecutorProvider.java | 9 +++- .../src/main/resources/thingsboard.yml | 5 +- .../provider/PubSubMonolithQueueFactory.java | 50 ++++++++----------- .../provider/PubSubTbCoreQueueFactory.java | 38 ++++++-------- .../PubSubTbRuleEngineQueueFactory.java | 28 +++++------ .../PubSubTbVersionControlQueueFactory.java | 13 ++--- .../provider/PubSubTransportQueueFactory.java | 20 +++----- .../pubsub/TbPubSubConsumerTemplate.java | 4 +- .../pubsub/TbPubSubProducerTemplate.java | 10 +--- .../pubsub/TbPubSubQueueExecutorProvider.java | 2 +- .../server/queue/pubsub/TbPubSubSettings.java | 2 + .../src/main/resources/tb-vc-executor.yml | 2 +- .../rule/engine/gcp/pubsub/TbPubSubNode.java | 6 +-- .../src/main/resources/tb-coap-transport.yml | 2 +- .../src/main/resources/tb-http-transport.yml | 2 +- .../src/main/resources/tb-lwm2m-transport.yml | 2 +- .../src/main/resources/tb-mqtt-transport.yml | 2 +- .../src/main/resources/tb-snmp-transport.yml | 2 +- 18 files changed, 88 insertions(+), 111 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/executors/PubSubRuleNodeExecutorProvider.java b/application/src/main/java/org/thingsboard/server/service/executors/PubSubRuleNodeExecutorProvider.java index f730949f45..520b8f855c 100644 --- a/application/src/main/java/org/thingsboard/server/service/executors/PubSubRuleNodeExecutorProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/executors/PubSubRuleNodeExecutorProvider.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.executors; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.thingsboard.common.util.ExecutorProvider; import org.thingsboard.common.util.ThingsBoardThreadFactory; @@ -27,6 +28,9 @@ import java.util.concurrent.ScheduledExecutorService; @Component public class PubSubRuleNodeExecutorProvider implements ExecutorProvider { + @Value("${service.rule_engine.pubsub.executor_thread_pool_size}") + private Integer threadPoolSize; + /** * Refers to com.google.cloud.pubsub.v1.Publisher default executor configuration */ @@ -35,7 +39,10 @@ public class PubSubRuleNodeExecutorProvider implements ExecutorProvider { @PostConstruct public void init() { - executor = Executors.newScheduledThreadPool(THREADS_PER_CPU * Runtime.getRuntime().availableProcessors(), ThingsBoardThreadFactory.forName("pubsub-rule-nodes"));; + if (threadPoolSize == null) { + threadPoolSize = THREADS_PER_CPU * Runtime.getRuntime().availableProcessors(); + } + executor = Executors.newScheduledThreadPool(threadPoolSize, ThingsBoardThreadFactory.forName("pubsub-rule-nodes")); } @Override diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 802756c105..304ad96a34 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1411,7 +1411,7 @@ queue: max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - # Number of threads of pubsub executor provider + # Thread pool size for pubsub queue executor provider. If not set - default pubsub executor provider value will be used (5 * number of available processors) executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again @@ -1641,6 +1641,9 @@ service: # Comma-separated list of tenant profile ids assigned to this Rule Engine. # This Rule Engine will only be responsible for tenants with these profiles (in case 'isolation' option is enabled in the profile). assigned_tenant_profiles: "${TB_RULE_ENGINE_ASSIGNED_TENANT_PROFILES:}" + pubsub: + # Thread pool size for pubsub rule node executor provider. If not set - default pubsub executor provider value will be used (5 * number of available processors) + executor_thread_pool_size: "${TB_RULE_ENGINE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" # Metrics parameters metrics: diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index 8ecd6564c8..554d452556 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -44,7 +44,6 @@ import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubQueueExecutorProvider; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; @@ -78,7 +77,6 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin vcAdmin; - private final TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider; public PubSubMonolithQueueFactory(TbPubSubSettings pubSubSettings, TbQueueCoreSettings coreSettings, @@ -89,8 +87,7 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng TbServiceInfoProvider serviceInfoProvider, TbPubSubSubscriptionSettings pubSubSubscriptionSettings, TbQueueRemoteJsInvokeSettings jsInvokeSettings, - TbQueueVersionControlSettings vcSettings, - TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider) { + TbQueueVersionControlSettings vcSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; @@ -99,7 +96,6 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng this.topicService = topicService; this.serviceInfoProvider = serviceInfoProvider; this.vcSettings = vcSettings; - this.tbPubSubQueueExecutorProvider = tbPubSubQueueExecutorProvider; this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); @@ -113,91 +109,85 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, transportNotificationSettings.getNotificationsTopic(), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, transportNotificationSettings.getNotificationsTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic(), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, ruleEngineSettings.getTopic(), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, coreSettings.getTopic(), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToVersionControlMsgConsumer() { return new TbPubSubConsumerTemplate<>(vcAdmin, pubSubSettings, vcSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()), - tbPubSubQueueExecutorProvider + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) ); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, configuration.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()), - tbPubSubQueueExecutorProvider); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, topicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()), - tbPubSubQueueExecutorProvider); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()), - tbPubSubQueueExecutorProvider); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, topicService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()), - tbPubSubQueueExecutorProvider); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders()), - tbPubSubQueueExecutorProvider); + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getResponsesTopic(), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getResponsesTopic()); } @Override @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic(), tbPubSubQueueExecutorProvider); + TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic()); TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), msg -> { RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder(); JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }, tbPubSubQueueExecutorProvider); + }); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); @@ -213,28 +203,28 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng @Override public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()), tbPubSubQueueExecutorProvider); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToOtaPackageStateServiceMsgConsumer() { return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getOtaPackageTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToOtaPackageStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders()), tbPubSubQueueExecutorProvider); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToOtaPackageStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getOtaPackageTopic(), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getOtaPackageTopic()); } @Override public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic(), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); } @Override public TbQueueProducer> createVersionControlMsgProducer() { - return new TbPubSubProducerTemplate<>(vcAdmin, pubSubSettings, vcSettings.getTopic(), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(vcAdmin, pubSubSettings, vcSettings.getTopic()); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 5443339d2a..3b45992e25 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -42,7 +42,6 @@ import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubQueueExecutorProvider; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; @@ -73,7 +72,6 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin ruleEngineAdmin; - private final TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider; public PubSubTbCoreQueueFactory(TbPubSubSettings pubSubSettings, TbQueueCoreSettings coreSettings, @@ -83,8 +81,7 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbQueueRuleEngineSettings ruleEngineSettings, - TbPubSubSubscriptionSettings pubSubSubscriptionSettings, - TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider) { + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.transportApiSettings = transportApiSettings; @@ -93,7 +90,6 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { this.jsInvokeSettings = jsInvokeSettings; this.transportNotificationSettings = transportNotificationSettings; this.ruleEngineSettings = ruleEngineSettings; - this.tbPubSubQueueExecutorProvider = tbPubSubQueueExecutorProvider; this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); @@ -104,64 +100,64 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, transportNotificationSettings.getNotificationsTopic(), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, transportNotificationSettings.getNotificationsTopic()); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, ruleEngineSettings.getTopic(), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, ruleEngineSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, coreSettings.getTopic(), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, coreSettings.getTopic()); } @Override public TbQueueConsumer> createToCoreMsgConsumer() { return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders()), tbPubSubQueueExecutorProvider); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, topicService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()), tbPubSubQueueExecutorProvider); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createTransportApiRequestConsumer() { return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getRequestsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders()), tbPubSubQueueExecutorProvider); + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createTransportApiResponseProducer() { - return new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getResponsesTopic(), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, transportApiSettings.getResponsesTopic()); } @Override @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic(), tbPubSubQueueExecutorProvider); + TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic()); TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), msg -> { JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }, tbPubSubQueueExecutorProvider); + }); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); @@ -177,25 +173,23 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders()), - tbPubSubQueueExecutorProvider); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToOtaPackageStateServiceMsgConsumer() { return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getOtaPackageTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToOtaPackageStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders()), - tbPubSubQueueExecutorProvider); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToOtaPackageStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getOtaPackageTopic(), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getOtaPackageTopic()); } @Override public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic(), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java index b0fde82e7d..a60f77852f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -40,7 +40,6 @@ import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubQueueExecutorProvider; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; @@ -63,7 +62,6 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -77,8 +75,7 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory TbServiceInfoProvider serviceInfoProvider, TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbQueueTransportNotificationSettings transportNotificationSettings, - TbPubSubSubscriptionSettings pubSubSubscriptionSettings, - TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider) { + TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; @@ -86,7 +83,6 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory this.serviceInfoProvider = serviceInfoProvider; this.jsInvokeSettings = jsInvokeSettings; this.transportNotificationSettings = transportNotificationSettings; - this.tbPubSubQueueExecutorProvider = tbPubSubQueueExecutorProvider; this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); @@ -96,53 +92,53 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic()), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic())); } @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, topicService.buildTopicName(ruleEngineSettings.getTopic()), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); } @Override public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(ruleEngineSettings.getTopic()), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic()), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic())); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic()), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic())); } @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, topicService.buildTopicName(configuration.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders()), tbPubSubQueueExecutorProvider); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, topicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders()), tbPubSubQueueExecutorProvider); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override @Bean public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic(), tbPubSubQueueExecutorProvider); + TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic()); TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), msg -> { JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }, tbPubSubQueueExecutorProvider); + }); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); @@ -157,12 +153,12 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory @Override public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic()), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); } @Override public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic()), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java index 29207ed429..8154d04d6d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java @@ -26,7 +26,6 @@ import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubQueueExecutorProvider; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; @@ -42,7 +41,6 @@ public class PubSubTbVersionControlQueueFactory implements TbVersionControlQueue private final TbQueueCoreSettings coreSettings; private final TbQueueVersionControlSettings vcSettings; private final TopicService topicService; - private final TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin notificationAdmin; @@ -52,14 +50,12 @@ public class PubSubTbVersionControlQueueFactory implements TbVersionControlQueue TbQueueCoreSettings coreSettings, TbQueueVersionControlSettings vcSettings, TbPubSubSubscriptionSettings pubSubSubscriptionSettings, - TopicService topicService, - TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider + TopicService topicService ) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.vcSettings = vcSettings; this.topicService = topicService; - this.tbPubSubQueueExecutorProvider = tbPubSubQueueExecutorProvider; this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); @@ -68,19 +64,18 @@ public class PubSubTbVersionControlQueueFactory implements TbVersionControlQueue @Override public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic()), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic()), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic())); } @Override public TbQueueConsumer> createToVersionControlMsgConsumer() { return new TbPubSubConsumerTemplate<>(vcAdmin, pubSubSettings, topicService.buildTopicName(vcSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()), - tbPubSubQueueExecutorProvider + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) ); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java index 94a454dd5b..edcad873c4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java @@ -37,7 +37,6 @@ import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubQueueExecutorProvider; import org.thingsboard.server.queue.pubsub.TbPubSubSettings; import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; @@ -59,7 +58,6 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TopicService topicService; - private final TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -73,8 +71,7 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbPubSubSubscriptionSettings pubSubSubscriptionSettings, - TopicService topicService, - TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider) { + TopicService topicService) { this.pubSubSettings = pubSubSettings; this.serviceInfoProvider = serviceInfoProvider; this.coreSettings = coreSettings; @@ -82,7 +79,6 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.topicService = topicService; - this.tbPubSubQueueExecutorProvider = tbPubSubQueueExecutorProvider; this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); @@ -92,10 +88,10 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { @Override public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbQueueProducer> producer = new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, topicService.buildTopicName(transportApiSettings.getRequestsTopic()), tbPubSubQueueExecutorProvider); + TbQueueProducer> producer = new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, topicService.buildTopicName(transportApiSettings.getRequestsTopic())); TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, topicService.buildTopicName(transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders()), tbPubSubQueueExecutorProvider); + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); @@ -110,29 +106,29 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { @Override public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, topicService.buildTopicName(ruleEngineSettings.getTopic()), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); } @Override public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic()), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic())); } @Override public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic()), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic())); } @Override public TbQueueConsumer> createTransportNotificationsConsumer() { return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders()), tbPubSubQueueExecutorProvider); + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic()), tbPubSubQueueExecutorProvider); + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java index fd01696873..1135053a66 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java @@ -63,7 +63,7 @@ public class TbPubSubConsumerTemplate extends AbstractPara private final SubscriberStub subscriber; private volatile int messagesPerTopic; - public TbPubSubConsumerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String topic, TbQueueMsgDecoder decoder, TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider) { + public TbPubSubConsumerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String topic, TbQueueMsgDecoder decoder) { super(topic); this.admin = admin; this.pubSubSettings = pubSubSettings; @@ -77,7 +77,7 @@ public class TbPubSubConsumerTemplate extends AbstractPara SubscriberStubSettings.defaultGrpcTransportProviderBuilder() .setMaxInboundMessageSize(pubSubSettings.getMaxMsgSize()) .build()) - .setExecutorProvider(FixedExecutorProvider.create(tbPubSubQueueExecutorProvider.getExecutor())) + .setExecutorProvider(FixedExecutorProvider.create(pubSubSettings.getTbPubSubQueueExecutorProvider().getExecutor())) .build(); this.subscriber = GrpcSubscriberStub.create(subscriberStubSettings); } catch (IOException e) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java index 48a104f20a..bf297d8af4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java @@ -25,8 +25,6 @@ import com.google.protobuf.ByteString; import com.google.pubsub.v1.ProjectTopicName; import com.google.pubsub.v1.PubsubMessage; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.common.util.ExecutorProvider; -import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueCallback; @@ -39,7 +37,6 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @Slf4j @@ -54,14 +51,11 @@ public class TbPubSubProducerTemplate implements TbQueuePr private final Map publisherMap = new ConcurrentHashMap<>(); private final ExecutorService pubExecutor = Executors.newCachedThreadPool(); - private final FixedExecutorProvider fixedExecutorProvider; - public TbPubSubProducerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String defaultTopic, ExecutorProvider executorProvider) { + public TbPubSubProducerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String defaultTopic) { this.defaultTopic = defaultTopic; this.admin = admin; this.pubSubSettings = pubSubSettings; - - fixedExecutorProvider = FixedExecutorProvider.create(executorProvider.getExecutor()); } @Override @@ -129,7 +123,7 @@ public class TbPubSubProducerTemplate implements TbQueuePr ProjectTopicName topicName = ProjectTopicName.of(pubSubSettings.getProjectId(), topic); Publisher publisher = Publisher.newBuilder(topicName) .setCredentialsProvider(pubSubSettings.getCredentialsProvider()) - .setExecutorProvider(fixedExecutorProvider) + .setExecutorProvider(FixedExecutorProvider.create(pubSubSettings.getTbPubSubQueueExecutorProvider().getExecutor())) .build(); publisherMap.put(topic, publisher); return publisher; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubQueueExecutorProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubQueueExecutorProvider.java index 4de75faa56..5e5f19651c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubQueueExecutorProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubQueueExecutorProvider.java @@ -43,7 +43,7 @@ public class TbPubSubQueueExecutorProvider implements ExecutorProvider { if (threadPoolSize == null) { threadPoolSize = THREADS_PER_CPU * Runtime.getRuntime().availableProcessors(); } - executor = Executors.newScheduledThreadPool(threadPoolSize, ThingsBoardThreadFactory.forName("pubsub-queue-executor"));; + executor = Executors.newScheduledThreadPool(threadPoolSize, ThingsBoardThreadFactory.forName("pubsub-queue-executor")); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java index fe9d85c306..530453f26b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java @@ -46,6 +46,8 @@ public class TbPubSubSettings { @Value("${queue.pubsub.max_messages}") private int maxMessages; + private final TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider; + private CredentialsProvider credentialsProvider; @PostConstruct diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 9fc9b884f0..62957debcf 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -172,7 +172,7 @@ queue: max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - # Number of threads of pubsub executor provider + # Thread pool size for pubsub queue executor provider. If not set - default pubsub executor provider value will be used (5 * number of available processors) executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" queue-properties: # Pub/Sub properties for Core subscribers, messages which will commit after ackDeadlineInSec period can be consumed again diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java index 39ab7bf525..3067947445 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/gcp/pubsub/TbPubSubNode.java @@ -69,7 +69,7 @@ public class TbPubSubNode extends TbAbstractExternalNode { super.init(ctx); this.config = TbNodeUtils.convert(configuration, TbPubSubNodeConfiguration.class); try { - this.pubSubClient = initPubSubClient(FixedExecutorProvider.create(ctx.getPubSubRuleNodeExecutorProvider().getExecutor())); + this.pubSubClient = initPubSubClient(ctx); } catch (Exception e) { throw new TbNodeException(e); } @@ -129,7 +129,7 @@ public class TbPubSubNode extends TbAbstractExternalNode { return TbMsg.transformMsgMetadata(origMsg, metaData); } - private Publisher initPubSubClient(FixedExecutorProvider fixedExecutorProvider) throws IOException { + private Publisher initPubSubClient(TbContext ctx) throws IOException { ProjectTopicName topicName = ProjectTopicName.of(config.getProjectId(), config.getTopicName()); ServiceAccountCredentials credentials = ServiceAccountCredentials.fromStream( @@ -149,7 +149,7 @@ public class TbPubSubNode extends TbAbstractExternalNode { return Publisher.newBuilder(topicName) .setCredentialsProvider(credProvider) .setRetrySettings(retrySettings) - .setExecutorProvider(fixedExecutorProvider) + .setExecutorProvider(FixedExecutorProvider.create(ctx.getPubSubRuleNodeExecutorProvider().getExecutor())) .build(); } } diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 27e5c8b273..c376d6a176 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -295,7 +295,7 @@ queue: max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - # Number of threads of pubsub executor provider + # Thread pool size for pubsub queue executor provider. If not set - default pubsub executor provider value will be used (5 * number of available processors) executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index b8070f6575..f3fd100fd7 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -278,7 +278,7 @@ queue: max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" # Number of messages per a consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - # Number of threads of pubsub executor provider + # Thread pool size for pubsub queue executor provider. If not set - default pubsub executor provider value will be used (5 * number of available processors) executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consume again diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index b1d40c720f..968385b5e2 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -374,7 +374,7 @@ queue: max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - # Number of threads of pubsub executor provider + # Thread pool size for pubsub queue executor provider. If not set - default pubsub executor provider value will be used (5 * number of available processors) executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 4d130eecd9..1e97c59253 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -311,7 +311,7 @@ queue: max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - # Number of threads of pubsub executor provider + # Thread pool size for pubsub queue executor provider. If not set - default pubsub executor provider value will be used (5 * number of available processors) executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index 2b89065a55..8d3f842325 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -264,7 +264,7 @@ queue: max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - # Number of threads of pubsub executor provider + # Thread pool size for pubsub queue executor provider. If not set - default pubsub executor provider value will be used (5 * number of available processors) executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again From 66b768a499bb299960871fe56ae9a0da2f79f1cb Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 17 Nov 2023 14:45:38 +0200 Subject: [PATCH 10/17] added @Lazy to PubSubRuleNodeExecutorProvider --- .../org/thingsboard/server/actors/ActorSystemContext.java | 1 + .../service/executors/PubSubRuleNodeExecutorProvider.java | 6 +++++- .../server/queue/pubsub/TbPubSubQueueExecutorProvider.java | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 24a240661f..607d5d8791 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -323,6 +323,7 @@ public class ActorSystemContext { @Getter private NotificationExecutorService notificationExecutor; + @Lazy @Autowired @Getter private PubSubRuleNodeExecutorProvider pubSubRuleNodeExecutorProvider; diff --git a/application/src/main/java/org/thingsboard/server/service/executors/PubSubRuleNodeExecutorProvider.java b/application/src/main/java/org/thingsboard/server/service/executors/PubSubRuleNodeExecutorProvider.java index 520b8f855c..74e1222acf 100644 --- a/application/src/main/java/org/thingsboard/server/service/executors/PubSubRuleNodeExecutorProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/executors/PubSubRuleNodeExecutorProvider.java @@ -16,15 +16,19 @@ package org.thingsboard.server.service.executors; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.common.util.ExecutorProvider; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +@Lazy +@TbRuleEngineComponent @Component public class PubSubRuleNodeExecutorProvider implements ExecutorProvider { @@ -53,7 +57,7 @@ public class PubSubRuleNodeExecutorProvider implements ExecutorProvider { @PreDestroy private void destroy() { if (executor != null) { - executor.shutdown(); + executor.shutdownNow(); } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubQueueExecutorProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubQueueExecutorProvider.java index 5e5f19651c..370670d554 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubQueueExecutorProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubQueueExecutorProvider.java @@ -54,7 +54,7 @@ public class TbPubSubQueueExecutorProvider implements ExecutorProvider { @PreDestroy private void destroy() { if (executor != null) { - executor.shutdown(); + executor.shutdownNow(); } } } From fa12ba544d3f35bd01afa169728e8881054c8222 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 20 Nov 2023 15:20:40 +0200 Subject: [PATCH 11/17] moved pubsub queue executor to PubSubSettings --- .../src/main/resources/thingsboard.yml | 6 +- .../pubsub/TbPubSubConsumerTemplate.java | 2 +- .../pubsub/TbPubSubProducerTemplate.java | 2 +- .../pubsub/TbPubSubQueueExecutorProvider.java | 60 ------------------- .../server/queue/pubsub/TbPubSubSettings.java | 25 +++++++- .../src/main/resources/tb-vc-executor.yml | 2 +- .../src/main/resources/tb-coap-transport.yml | 2 +- .../src/main/resources/tb-http-transport.yml | 2 +- .../src/main/resources/tb-lwm2m-transport.yml | 2 +- .../src/main/resources/tb-mqtt-transport.yml | 2 +- .../src/main/resources/tb-snmp-transport.yml | 2 +- 11 files changed, 35 insertions(+), 72 deletions(-) delete mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubQueueExecutorProvider.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 304ad96a34..6025114a3d 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1411,8 +1411,8 @@ queue: max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - # Thread pool size for pubsub queue executor provider. If not set - default pubsub executor provider value will be used (5 * number of available processors) - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" + # Thread pool size for pubsub queue executor provider. If set to 0 - default pubsub executor provider value will be used (5 * number of available processors) + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:0}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" @@ -1643,7 +1643,7 @@ service: assigned_tenant_profiles: "${TB_RULE_ENGINE_ASSIGNED_TENANT_PROFILES:}" pubsub: # Thread pool size for pubsub rule node executor provider. If not set - default pubsub executor provider value will be used (5 * number of available processors) - executor_thread_pool_size: "${TB_RULE_ENGINE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" + executor_thread_pool_size: "${TB_RULE_ENGINE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:0}" # Metrics parameters metrics: diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java index 1135053a66..1a84b80cd2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java @@ -77,7 +77,7 @@ public class TbPubSubConsumerTemplate extends AbstractPara SubscriberStubSettings.defaultGrpcTransportProviderBuilder() .setMaxInboundMessageSize(pubSubSettings.getMaxMsgSize()) .build()) - .setExecutorProvider(FixedExecutorProvider.create(pubSubSettings.getTbPubSubQueueExecutorProvider().getExecutor())) + .setExecutorProvider(pubSubSettings.getExecutorProvider()) .build(); this.subscriber = GrpcSubscriberStub.create(subscriberStubSettings); } catch (IOException e) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java index bf297d8af4..3107855ab1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java @@ -123,7 +123,7 @@ public class TbPubSubProducerTemplate implements TbQueuePr ProjectTopicName topicName = ProjectTopicName.of(pubSubSettings.getProjectId(), topic); Publisher publisher = Publisher.newBuilder(topicName) .setCredentialsProvider(pubSubSettings.getCredentialsProvider()) - .setExecutorProvider(FixedExecutorProvider.create(pubSubSettings.getTbPubSubQueueExecutorProvider().getExecutor())) + .setExecutorProvider(pubSubSettings.getExecutorProvider()) .build(); publisherMap.put(topic, publisher); return publisher; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubQueueExecutorProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubQueueExecutorProvider.java deleted file mode 100644 index 370670d554..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubQueueExecutorProvider.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright © 2016-2023 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.pubsub; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.ExecutorProvider; -import org.thingsboard.common.util.ThingsBoardThreadFactory; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; - -@ConditionalOnExpression("'${queue.type:null}'=='pubsub'") -@Component -public class TbPubSubQueueExecutorProvider implements ExecutorProvider { - - @Value("${queue.pubsub.executor_thread_pool_size}") - private Integer threadPoolSize; - /** - * Refers to com.google.cloud.pubsub.v1.Publisher default executor configuration - */ - private static final int THREADS_PER_CPU = 5; - private ScheduledExecutorService executor; - - @PostConstruct - public void init() { - if (threadPoolSize == null) { - threadPoolSize = THREADS_PER_CPU * Runtime.getRuntime().availableProcessors(); - } - executor = Executors.newScheduledThreadPool(threadPoolSize, ThingsBoardThreadFactory.forName("pubsub-queue-executor")); - } - - @Override - public ScheduledExecutorService getExecutor() { - return executor; - } - - @PreDestroy - private void destroy() { - if (executor != null) { - executor.shutdownNow(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java index 530453f26b..145661fed1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java @@ -17,16 +17,20 @@ package org.thingsboard.server.queue.pubsub; import com.google.api.gax.core.CredentialsProvider; import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.api.gax.core.FixedExecutorProvider; import com.google.auth.oauth2.ServiceAccountCredentials; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.util.concurrent.Executors; @Slf4j @ConditionalOnExpression("'${queue.type:null}'=='pubsub'") @@ -46,7 +50,15 @@ public class TbPubSubSettings { @Value("${queue.pubsub.max_messages}") private int maxMessages; - private final TbPubSubQueueExecutorProvider tbPubSubQueueExecutorProvider; + @Value("${queue.pubsub.executor_thread_pool_size:0}") + private int threadPoolSize; + + /** + * Refers to com.google.cloud.pubsub.v1.Publisher default executor configuration + */ + private static final int THREADS_PER_CPU = 5; + + private FixedExecutorProvider executorProvider; private CredentialsProvider credentialsProvider; @@ -55,6 +67,17 @@ public class TbPubSubSettings { ServiceAccountCredentials credentials = ServiceAccountCredentials.fromStream( new ByteArrayInputStream(serviceAccount.getBytes())); credentialsProvider = FixedCredentialsProvider.create(credentials); + if (threadPoolSize == 0) { + threadPoolSize = THREADS_PER_CPU * Runtime.getRuntime().availableProcessors(); + } + executorProvider = FixedExecutorProvider + .create(Executors.newScheduledThreadPool(threadPoolSize, ThingsBoardThreadFactory.forName("pubsub-queue-executor"))); } + @PreDestroy + private void destroy() { + if (executorProvider != null) { + executorProvider.getExecutor().shutdownNow(); + } + } } diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 62957debcf..8c363b019c 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -173,7 +173,7 @@ queue: # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" # Thread pool size for pubsub queue executor provider. If not set - default pubsub executor provider value will be used (5 * number of available processors) - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:0}" queue-properties: # Pub/Sub properties for Core subscribers, messages which will commit after ackDeadlineInSec period can be consumed again core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index c376d6a176..0db3e71f52 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -296,7 +296,7 @@ queue: # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" # Thread pool size for pubsub queue executor provider. If not set - default pubsub executor provider value will be used (5 * number of available processors) - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:0}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index f3fd100fd7..d4fda384c7 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -279,7 +279,7 @@ queue: # Number of messages per a consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" # Thread pool size for pubsub queue executor provider. If not set - default pubsub executor provider value will be used (5 * number of available processors) - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:0}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consume again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index 968385b5e2..06b8c1a84a 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -375,7 +375,7 @@ queue: # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" # Thread pool size for pubsub queue executor provider. If not set - default pubsub executor provider value will be used (5 * number of available processors) - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:0}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 1e97c59253..afb34d7063 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -312,7 +312,7 @@ queue: # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" # Thread pool size for pubsub queue executor provider. If not set - default pubsub executor provider value will be used (5 * number of available processors) - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:0}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index 8d3f842325..6ee6333fc2 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -265,7 +265,7 @@ queue: # Number of messages per consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" # Thread pool size for pubsub queue executor provider. If not set - default pubsub executor provider value will be used (5 * number of available processors) - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:}" + executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:0}" queue-properties: # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" From 50c0822e05978ddcd7dbfacc3774c07e48a14f34 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 29 Nov 2023 11:33:25 +0200 Subject: [PATCH 12/17] updated postgres image to from 12 to 15 version --- .../data/json/edge/install_instructions/docker/instructions.md | 2 +- docker/docker-compose.hybrid.yml | 2 +- docker/docker-compose.postgres.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/application/src/main/data/json/edge/install_instructions/docker/instructions.md b/application/src/main/data/json/edge/install_instructions/docker/instructions.md index e308a38076..d8f9731889 100644 --- a/application/src/main/data/json/edge/install_instructions/docker/instructions.md +++ b/application/src/main/data/json/edge/install_instructions/docker/instructions.md @@ -51,7 +51,7 @@ services: - ~/.mytb-edge-logs:/var/log/tb-edge postgres: restart: always - image: "postgres:12" + image: "postgres:15" ports: - "5432" environment: diff --git a/docker/docker-compose.hybrid.yml b/docker/docker-compose.hybrid.yml index 29cd65a127..7f952fe6eb 100644 --- a/docker/docker-compose.hybrid.yml +++ b/docker/docker-compose.hybrid.yml @@ -19,7 +19,7 @@ version: '3.0' services: postgres: restart: always - image: "postgres:12" + image: "postgres:15" ports: - "5432" environment: diff --git a/docker/docker-compose.postgres.yml b/docker/docker-compose.postgres.yml index 7f5f69a61f..218642317b 100644 --- a/docker/docker-compose.postgres.yml +++ b/docker/docker-compose.postgres.yml @@ -19,7 +19,7 @@ version: '3.0' services: postgres: restart: always - image: "postgres:12" + image: "postgres:15" ports: - "5432" environment: From 4627bde4af23351ffeb74ff16e30aec8f6e6fd4a Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 29 Nov 2023 15:18:01 +0200 Subject: [PATCH 13/17] updated postgres version from 12 to 15 --- msa/tb/docker-cassandra/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msa/tb/docker-cassandra/Dockerfile b/msa/tb/docker-cassandra/Dockerfile index 2f23aec399..0a0c36ed88 100644 --- a/msa/tb/docker-cassandra/Dockerfile +++ b/msa/tb/docker-cassandra/Dockerfile @@ -16,7 +16,7 @@ FROM thingsboard/openjdk11:bullseye-slim -ENV PG_MAJOR=12 +ENV PG_MAJOR=15 ENV DATA_FOLDER=/data From ba2e877a08e7115ebc0328a3bd7bd3c0269845e3 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 30 Nov 2023 13:06:44 +0200 Subject: [PATCH 14/17] refactoring: moved proto files and converter classes to separate module - proto --- .../controller/TelemetryController.java | 2 +- .../constructor/EntityDataMsgConstructor.java | 2 +- .../telemetry/BaseTelemetryProcessor.java | 2 +- .../queue/DefaultTbClusterService.java | 2 +- .../queue/DefaultTbCoreConsumerService.java | 1 + .../DefaultTbRuleEngineConsumerService.java | 1 + .../csv/AbstractBulkImportService.java | 2 +- .../server/edge/DeviceEdgeTest.java | 2 +- common/cluster-api/pom.xml | 4 + common/pom.xml | 1 + common/proto/pom.xml | 109 ++++++++++++++++++ .../common}/adaptor/AdaptorException.java | 2 +- .../server/common}/adaptor/JsonConverter.java | 2 +- .../common}/adaptor/JsonConverterConfig.java | 2 +- .../common}/adaptor/ProtoConverter.java | 2 +- .../server/common/util}/ProtoUtils.java | 2 +- .../src/main/proto/jsinvoke.proto | 0 .../src/main/proto/queue.proto | 0 .../src/main/proto/transport.proto | 0 .../common/adaptor}/JsonConverterTest.java | 3 +- .../server/common/util}/ProtoUtilsTest.java | 2 +- .../transport/coap/CoapTransportResource.java | 4 +- .../coap/adaptors/CoapAdaptorUtils.java | 2 +- .../coap/adaptors/CoapTransportAdaptor.java | 2 +- .../coap/adaptors/JsonCoapAdaptor.java | 4 +- .../coap/adaptors/ProtoCoapAdaptor.java | 6 +- .../GetAttributesSyncSessionCallback.java | 2 +- .../ToServerRpcSyncSessionCallback.java | 2 +- .../coap/client/CoapClientContext.java | 2 +- .../coap/client/DefaultCoapClientContext.java | 2 +- .../efento/CoapEfentoTransportResource.java | 5 +- .../efento/adaptor/EfentoCoapAdaptor.java | 4 +- .../transport/http/DeviceApiController.java | 2 +- .../server/adaptors/LwM2MJsonAdaptor.java | 4 +- .../adaptors/LwM2MTransportAdaptor.java | 2 +- .../transport/mqtt/MqttTransportHandler.java | 2 +- .../BackwardCompatibilityAdaptor.java | 2 +- .../mqtt/adaptors/JsonMqttAdaptor.java | 4 +- .../mqtt/adaptors/MqttTransportAdaptor.java | 2 +- .../mqtt/adaptors/ProtoMqttAdaptor.java | 6 +- .../AbstractGatewaySessionHandler.java | 6 +- .../mqtt/session/GatewaySessionHandler.java | 2 +- .../session/SparkplugNodeSessionHandler.java | 6 +- .../mqtt/util/sparkplug/MetricDataType.java | 2 +- .../snmp/service/SnmpTransportService.java | 2 +- pom.xml | 5 + .../TbCopyAttributesToEntityViewNode.java | 2 +- .../rule/engine/profile/AlarmRuleState.java | 2 +- .../rule/engine/profile/DeviceState.java | 2 +- .../engine/telemetry/TbMsgAttributesNode.java | 2 +- .../engine/telemetry/TbMsgTimeseriesNode.java | 2 +- 51 files changed, 177 insertions(+), 58 deletions(-) create mode 100644 common/proto/pom.xml rename common/{transport/transport-api/src/main/java/org/thingsboard/server/common/transport => proto/src/main/java/org/thingsboard/server/common}/adaptor/AdaptorException.java (94%) rename common/{transport/transport-api/src/main/java/org/thingsboard/server/common/transport => proto/src/main/java/org/thingsboard/server/common}/adaptor/JsonConverter.java (99%) rename common/{transport/transport-api/src/main/java/org/thingsboard/server/common/transport => proto/src/main/java/org/thingsboard/server/common}/adaptor/JsonConverterConfig.java (96%) rename common/{transport/transport-api/src/main/java/org/thingsboard/server/common/transport => proto/src/main/java/org/thingsboard/server/common}/adaptor/ProtoConverter.java (99%) rename {application/src/main/java/org/thingsboard/server/service/queue => common/proto/src/main/java/org/thingsboard/server/common/util}/ProtoUtils.java (99%) rename common/{cluster-api => proto}/src/main/proto/jsinvoke.proto (100%) rename common/{cluster-api => proto}/src/main/proto/queue.proto (100%) rename common/{transport/transport-api => proto}/src/main/proto/transport.proto (100%) rename common/{transport/transport-api/src/test/java => proto/src/test/java/org/thingsboard/server/common/adaptor}/JsonConverterTest.java (98%) rename {application/src/test/java/org/thingsboard/server/service/queue => common/proto/src/test/java/org/thingsboard/server/common/util}/ProtoUtilsTest.java (99%) diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index 5f88752ff4..55f4f3afbd 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -74,7 +74,7 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; +import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.exception.InvalidParametersException; import org.thingsboard.server.exception.UncheckedApiException; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/EntityDataMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/EntityDataMsgConstructor.java index 47dbad231d..a8ff4ffa1a 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/EntityDataMsgConstructor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/EntityDataMsgConstructor.java @@ -28,7 +28,7 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; +import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.gen.edge.v1.AttributeDeleteMsg; import org.thingsboard.server.gen.edge.v1.EntityDataProto; import org.thingsboard.server.gen.transport.TransportProtos; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java index 974e9267f6..f6092a65c3 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/telemetry/BaseTelemetryProcessor.java @@ -54,7 +54,7 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; +import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.common.transport.util.JsonUtils; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.gen.edge.v1.AttributeDeleteMsg; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index af2a54a7ac..14930912d1 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -81,7 +81,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; -import static org.thingsboard.server.service.queue.ProtoUtils.toProto; +import static org.thingsboard.server.common.util.ProtoUtils.toProto; @Service @Slf4j diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 754537ef11..1511e227d5 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -45,6 +45,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 12fa5d3f27..273d7afc97 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.rpc.RpcError; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; +import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java index c03dd8dcb7..30765bd959 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java @@ -47,7 +47,7 @@ import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportColumn import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportRequest; import org.thingsboard.server.common.data.sync.ie.importing.csv.BulkImportResult; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; +import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.controller.BaseController; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.service.action.EntityActionService; diff --git a/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java index e6fee370f1..2cafd68f4f 100644 --- a/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java @@ -51,7 +51,7 @@ import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.msg.session.FeatureType; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; +import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.gen.edge.v1.AttributesRequestMsg; import org.thingsboard.server.gen.edge.v1.DeviceCredentialsRequestMsg; diff --git a/common/cluster-api/pom.xml b/common/cluster-api/pom.xml index 6a0831de47..cd8f551aa7 100644 --- a/common/cluster-api/pom.xml +++ b/common/cluster-api/pom.xml @@ -40,6 +40,10 @@ org.thingsboard.common data + + org.thingsboard.common + proto + org.thingsboard.common message diff --git a/common/pom.xml b/common/pom.xml index b00b13db57..76387a1773 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -48,6 +48,7 @@ edge-api version-control script + proto diff --git a/common/proto/pom.xml b/common/proto/pom.xml new file mode 100644 index 0000000000..acaf94cd74 --- /dev/null +++ b/common/proto/pom.xml @@ -0,0 +1,109 @@ + + + 4.0.0 + + org.thingsboard + 3.6.2-SNAPSHOT + common + + org.thingsboard.common + proto + jar + + Thingsboard Server Common Protobuf and gRPC structures + https://thingsboard.io + + + UTF-8 + ${basedir}/../.. + + + + + org.thingsboard.common + data + + + org.thingsboard.common + message + + + com.google.protobuf + protobuf-java + + + com.google.protobuf + protobuf-java-util + + + io.grpc + grpc-netty-shaded + provided + + + io.grpc + grpc-protobuf + provided + + + io.grpc + grpc-stub + provided + + + org.springframework.boot + spring-boot-starter-web + provided + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.awaitility + awaitility + test + + + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + + + + + + + thingsboard-repo-deploy + ThingsBoard Repo Deployment + https://repo.thingsboard.io/artifactory/libs-release-public + + + + \ No newline at end of file diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/AdaptorException.java b/common/proto/src/main/java/org/thingsboard/server/common/adaptor/AdaptorException.java similarity index 94% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/AdaptorException.java rename to common/proto/src/main/java/org/thingsboard/server/common/adaptor/AdaptorException.java index 8b2908a4b5..1c22163c1f 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/AdaptorException.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/adaptor/AdaptorException.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport.adaptor; +package org.thingsboard.server.common.adaptor; public class AdaptorException extends Exception { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java b/common/proto/src/main/java/org/thingsboard/server/common/adaptor/JsonConverter.java similarity index 99% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java rename to common/proto/src/main/java/org/thingsboard/server/common/adaptor/JsonConverter.java index 51e74d5097..c51bc784a6 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/adaptor/JsonConverter.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport.adaptor; +package org.thingsboard.server.common.adaptor; import com.google.gson.Gson; import com.google.gson.JsonArray; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverterConfig.java b/common/proto/src/main/java/org/thingsboard/server/common/adaptor/JsonConverterConfig.java similarity index 96% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverterConfig.java rename to common/proto/src/main/java/org/thingsboard/server/common/adaptor/JsonConverterConfig.java index 0a9df150e5..e160c051f5 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverterConfig.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/adaptor/JsonConverterConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport.adaptor; +package org.thingsboard.server.common.adaptor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/ProtoConverter.java b/common/proto/src/main/java/org/thingsboard/server/common/adaptor/ProtoConverter.java similarity index 99% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/ProtoConverter.java rename to common/proto/src/main/java/org/thingsboard/server/common/adaptor/ProtoConverter.java index 54358709f0..098108c864 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/ProtoConverter.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/adaptor/ProtoConverter.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport.adaptor; +package org.thingsboard.server.common.adaptor; import com.google.gson.Gson; import com.google.gson.JsonElement; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java similarity index 99% rename from application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java rename to common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 946993151d..fd9e35b860 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue; +package org.thingsboard.server.common.util; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; diff --git a/common/cluster-api/src/main/proto/jsinvoke.proto b/common/proto/src/main/proto/jsinvoke.proto similarity index 100% rename from common/cluster-api/src/main/proto/jsinvoke.proto rename to common/proto/src/main/proto/jsinvoke.proto diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto similarity index 100% rename from common/cluster-api/src/main/proto/queue.proto rename to common/proto/src/main/proto/queue.proto diff --git a/common/transport/transport-api/src/main/proto/transport.proto b/common/proto/src/main/proto/transport.proto similarity index 100% rename from common/transport/transport-api/src/main/proto/transport.proto rename to common/proto/src/main/proto/transport.proto diff --git a/common/transport/transport-api/src/test/java/JsonConverterTest.java b/common/proto/src/test/java/org/thingsboard/server/common/adaptor/JsonConverterTest.java similarity index 98% rename from common/transport/transport-api/src/test/java/JsonConverterTest.java rename to common/proto/src/test/java/org/thingsboard/server/common/adaptor/JsonConverterTest.java index 39ed04a29a..b9f9819823 100644 --- a/common/transport/transport-api/src/test/java/JsonConverterTest.java +++ b/common/proto/src/test/java/org/thingsboard/server/common/adaptor/JsonConverterTest.java @@ -1,4 +1,4 @@ -/** +package org.thingsboard.server.common.adaptor; /** * Copyright © 2016-2023 The Thingsboard Authors * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,7 +19,6 @@ import org.junit.Assert; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; import java.util.ArrayList; diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ProtoUtilsTest.java b/common/proto/src/test/java/org/thingsboard/server/common/util/ProtoUtilsTest.java similarity index 99% rename from application/src/test/java/org/thingsboard/server/service/queue/ProtoUtilsTest.java rename to common/proto/src/test/java/org/thingsboard/server/common/util/ProtoUtilsTest.java index 1a696b730d..a41b2be208 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ProtoUtilsTest.java +++ b/common/proto/src/test/java/org/thingsboard/server/common/util/ProtoUtilsTest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue; +package org.thingsboard.server.common.util; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java index bd02d9fcc4..63979ce500 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java @@ -35,8 +35,8 @@ import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.security.DeviceTokenCredentials; import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; +import org.thingsboard.server.common.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.coap.callback.CoapDeviceAuthCallback; diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/CoapAdaptorUtils.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/CoapAdaptorUtils.java index 710f18a8b7..629e0380e0 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/CoapAdaptorUtils.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/CoapAdaptorUtils.java @@ -17,7 +17,7 @@ package org.thingsboard.server.transport.coap.adaptors; import org.eclipse.californium.core.coap.Request; import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.AdaptorException; import org.thingsboard.server.gen.transport.TransportProtos; import java.util.Arrays; diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/CoapTransportAdaptor.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/CoapTransportAdaptor.java index 05a38e1ad1..e88251c93c 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/CoapTransportAdaptor.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/CoapTransportAdaptor.java @@ -19,7 +19,7 @@ import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; import org.eclipse.californium.core.coap.Request; import org.eclipse.californium.core.coap.Response; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.AdaptorException; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceRequestMsg; diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/JsonCoapAdaptor.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/JsonCoapAdaptor.java index d81ff32edd..fffe1a7e1a 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/JsonCoapAdaptor.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/JsonCoapAdaptor.java @@ -29,8 +29,8 @@ import org.eclipse.californium.core.coap.Response; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; +import org.thingsboard.server.common.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.coap.CoapTransportResource; diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/ProtoCoapAdaptor.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/ProtoCoapAdaptor.java index 4d3b0015ef..5287645591 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/ProtoCoapAdaptor.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/ProtoCoapAdaptor.java @@ -28,9 +28,9 @@ import org.eclipse.californium.core.coap.Response; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; -import org.thingsboard.server.common.transport.adaptor.ProtoConverter; +import org.thingsboard.server.common.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.JsonConverter; +import org.thingsboard.server.common.adaptor.ProtoConverter; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.coap.CoapTransportResource; diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/callback/GetAttributesSyncSessionCallback.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/callback/GetAttributesSyncSessionCallback.java index 907e49f600..caacd31012 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/callback/GetAttributesSyncSessionCallback.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/callback/GetAttributesSyncSessionCallback.java @@ -20,7 +20,7 @@ import org.eclipse.californium.core.coap.CoAP; import org.eclipse.californium.core.coap.Request; import org.eclipse.californium.core.coap.Response; import org.eclipse.californium.core.server.resources.CoapExchange; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.AdaptorException; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.coap.client.TbCoapClientState; diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/callback/ToServerRpcSyncSessionCallback.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/callback/ToServerRpcSyncSessionCallback.java index 2209ea6613..fc35da0bd6 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/callback/ToServerRpcSyncSessionCallback.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/callback/ToServerRpcSyncSessionCallback.java @@ -19,7 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.eclipse.californium.core.coap.CoAP; import org.eclipse.californium.core.coap.Request; import org.eclipse.californium.core.server.resources.CoapExchange; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.AdaptorException; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.coap.client.TbCoapClientState; diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/client/CoapClientContext.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/client/CoapClientContext.java index a8531a7fba..a98a04ce28 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/client/CoapClientContext.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/client/CoapClientContext.java @@ -19,7 +19,7 @@ import org.eclipse.californium.core.observe.ObserveRelation; import org.eclipse.californium.core.server.resources.CoapExchange; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.transport.coap.CoapSessionMsgType; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.AdaptorException; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.gen.transport.TransportProtos; diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/client/DefaultCoapClientContext.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/client/DefaultCoapClientContext.java index 26effe7058..f4346946a7 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/client/DefaultCoapClientContext.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/client/DefaultCoapClientContext.java @@ -53,7 +53,7 @@ import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportDeviceProfileCache; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.AdaptorException; import org.thingsboard.server.common.transport.auth.SessionInfoCreator; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.gen.transport.TransportProtos; diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/efento/CoapEfentoTransportResource.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/efento/CoapEfentoTransportResource.java index 1c0a599082..b3304c6a97 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/efento/CoapEfentoTransportResource.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/efento/CoapEfentoTransportResource.java @@ -33,8 +33,8 @@ import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.EfentoCoapDeviceTypeConfiguration; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; -import org.thingsboard.server.common.transport.adaptor.ProtoConverter; +import org.thingsboard.server.common.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.ProtoConverter; import org.thingsboard.server.common.transport.auth.SessionInfoCreator; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.coap.ConfigProtos; @@ -56,7 +56,6 @@ import java.util.Map; import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; import static com.google.gson.JsonParser.parseString; import static org.thingsboard.server.transport.coap.CoapTransportService.CONFIGURATION; diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/efento/adaptor/EfentoCoapAdaptor.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/efento/adaptor/EfentoCoapAdaptor.java index ce75608a7d..fa6394d41b 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/efento/adaptor/EfentoCoapAdaptor.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/efento/adaptor/EfentoCoapAdaptor.java @@ -19,8 +19,8 @@ import com.google.gson.Gson; import com.google.gson.JsonElement; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; +import org.thingsboard.server.common.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.coap.efento.CoapEfentoTransportResource; diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java index 041ae0510d..589e93d803 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java @@ -46,7 +46,7 @@ import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportContext; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; +import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.common.transport.auth.SessionInfoCreator; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.gen.transport.TransportProtos; diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/adaptors/LwM2MJsonAdaptor.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/adaptors/LwM2MJsonAdaptor.java index feb74b47f9..afe1d160ce 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/adaptors/LwM2MJsonAdaptor.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/adaptors/LwM2MJsonAdaptor.java @@ -19,8 +19,8 @@ import com.google.gson.JsonElement; import com.google.gson.JsonSyntaxException; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; +import org.thingsboard.server.common.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbLwM2mTransportComponent; diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/adaptors/LwM2MTransportAdaptor.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/adaptors/LwM2MTransportAdaptor.java index 96e6008c63..fe24b21053 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/adaptors/LwM2MTransportAdaptor.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/adaptors/LwM2MTransportAdaptor.java @@ -16,7 +16,7 @@ package org.thingsboard.server.transport.lwm2m.server.adaptors; import com.google.gson.JsonElement; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.AdaptorException; import org.thingsboard.server.gen.transport.TransportProtos; import java.util.Collection; diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 3bafbb518e..87f6a47150 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -64,7 +64,7 @@ import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.AdaptorException; import org.thingsboard.server.common.transport.auth.SessionInfoCreator; import org.thingsboard.server.common.transport.auth.TransportDeviceInfo; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/BackwardCompatibilityAdaptor.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/BackwardCompatibilityAdaptor.java index 40be642a1a..73bba60cff 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/BackwardCompatibilityAdaptor.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/BackwardCompatibilityAdaptor.java @@ -21,7 +21,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.ota.OtaPackageType; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.AdaptorException; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.mqtt.session.MqttDeviceAwareSessionContext; diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java index 513fd97cf6..a30749b17d 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java @@ -30,8 +30,8 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.device.profile.MqttTopics; import org.thingsboard.server.common.data.ota.OtaPackageType; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; +import org.thingsboard.server.common.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.mqtt.session.MqttDeviceAwareSessionContext; diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/MqttTransportAdaptor.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/MqttTransportAdaptor.java index 3764c2a9a9..10599d0be7 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/MqttTransportAdaptor.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/MqttTransportAdaptor.java @@ -24,7 +24,7 @@ import io.netty.handler.codec.mqtt.MqttMessageType; import io.netty.handler.codec.mqtt.MqttPublishMessage; import io.netty.handler.codec.mqtt.MqttPublishVariableHeader; import org.thingsboard.server.common.data.ota.OtaPackageType; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.AdaptorException; import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/ProtoMqttAdaptor.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/ProtoMqttAdaptor.java index 39176fa212..3ff4faf8e8 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/ProtoMqttAdaptor.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/ProtoMqttAdaptor.java @@ -25,12 +25,12 @@ import io.netty.handler.codec.mqtt.MqttMessage; import io.netty.handler.codec.mqtt.MqttPublishMessage; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.JsonConverter; +import org.thingsboard.server.common.adaptor.ProtoConverter; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.device.profile.MqttTopics; import org.thingsboard.server.common.data.ota.OtaPackageType; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; -import org.thingsboard.server.common.transport.adaptor.ProtoConverter; import org.thingsboard.server.gen.transport.TransportApiProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.mqtt.session.DeviceSessionCtx; diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java index 8f964f8369..64aa448931 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java @@ -40,9 +40,9 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; -import org.thingsboard.server.common.transport.adaptor.ProtoConverter; +import org.thingsboard.server.common.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.JsonConverter; +import org.thingsboard.server.common.adaptor.ProtoConverter; import org.thingsboard.server.common.transport.auth.GetOrCreateDeviceFromGatewayResponse; import org.thingsboard.server.common.transport.auth.TransportDeviceInfo; import org.thingsboard.server.gen.transport.TransportApiProtos; diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java index de0f2a502b..6ff505cb36 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java @@ -17,7 +17,7 @@ package org.thingsboard.server.transport.mqtt.session; import io.netty.buffer.ByteBuf; import io.netty.handler.codec.mqtt.MqttPublishMessage; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.AdaptorException; import org.thingsboard.server.common.transport.auth.GetOrCreateDeviceFromGatewayResponse; import java.util.UUID; diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/SparkplugNodeSessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/SparkplugNodeSessionHandler.java index 48998d58cd..f3aaa2708f 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/SparkplugNodeSessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/SparkplugNodeSessionHandler.java @@ -32,9 +32,9 @@ import org.springframework.util.CollectionUtils; import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; -import org.thingsboard.server.common.transport.adaptor.ProtoConverter; +import org.thingsboard.server.common.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.JsonConverter; +import org.thingsboard.server.common.adaptor.ProtoConverter; import org.thingsboard.server.common.transport.auth.GetOrCreateDeviceFromGatewayResponse; import org.thingsboard.server.gen.transport.TransportApiProtos; import org.thingsboard.server.gen.transport.TransportProtos; diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/util/sparkplug/MetricDataType.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/util/sparkplug/MetricDataType.java index d5e781a402..c49286a4b1 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/util/sparkplug/MetricDataType.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/util/sparkplug/MetricDataType.java @@ -16,7 +16,7 @@ package org.thingsboard.server.transport.mqtt.util.sparkplug; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.AdaptorException; import org.thingsboard.server.gen.transport.mqtt.SparkplugBProto; import java.math.BigInteger; diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java index 0b60db68b1..014fcf8d67 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java @@ -53,7 +53,7 @@ import org.thingsboard.server.common.data.transport.snmp.SnmpMethod; import org.thingsboard.server.common.data.transport.snmp.config.RepeatingQueryingSnmpCommunicationConfig; import org.thingsboard.server.common.data.transport.snmp.config.SnmpCommunicationConfig; import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; +import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbSnmpTransportComponent; import org.thingsboard.server.transport.snmp.SnmpTransportContext; diff --git a/pom.xml b/pom.xml index 7fd3769704..e590888bc3 100755 --- a/pom.xml +++ b/pom.xml @@ -893,6 +893,11 @@ version-control ${project.version} + + org.thingsboard.common + proto + ${project.version} + org.thingsboard.common cache diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java index c103dd4006..af4c7bc0c2 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java @@ -36,7 +36,7 @@ import org.thingsboard.server.common.data.objects.AttributesEntityView; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; +import org.thingsboard.server.common.adaptor.JsonConverter; import javax.annotation.Nullable; import java.util.ArrayList; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java index f31896a8c9..6bb37438a1 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java @@ -41,7 +41,7 @@ import org.thingsboard.server.common.data.query.KeyFilterPredicate; import org.thingsboard.server.common.data.query.NumericFilterPredicate; import org.thingsboard.server.common.data.query.StringFilterPredicate; import org.thingsboard.server.common.msg.tools.SchedulerUtils; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; +import org.thingsboard.server.common.adaptor.JsonConverter; import java.time.Instant; import java.time.ZoneId; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DeviceState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DeviceState.java index 2aeb4eedf5..b38fd29af7 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DeviceState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DeviceState.java @@ -40,7 +40,7 @@ import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.rule.RuleNodeState; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; +import org.thingsboard.server.common.adaptor.JsonConverter; import org.thingsboard.server.dao.sql.query.EntityKeyMapping; import java.util.ArrayList; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index 9bbee45477..06498241fb 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -34,7 +34,7 @@ import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; +import org.thingsboard.server.common.adaptor.JsonConverter; import java.util.ArrayList; import java.util.List; diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 799089d4a8..8762839c07 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -31,7 +31,7 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; +import org.thingsboard.server.common.adaptor.JsonConverter; import java.util.ArrayList; import java.util.List; From a141b9b4ab2951192fe1d4c1b0c3a7c81bc08504 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 30 Nov 2023 13:28:33 +0200 Subject: [PATCH 15/17] refactoring --- .../thingsboard/server/common/adaptor/JsonConverterTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/proto/src/test/java/org/thingsboard/server/common/adaptor/JsonConverterTest.java b/common/proto/src/test/java/org/thingsboard/server/common/adaptor/JsonConverterTest.java index b9f9819823..9e4bf6d21b 100644 --- a/common/proto/src/test/java/org/thingsboard/server/common/adaptor/JsonConverterTest.java +++ b/common/proto/src/test/java/org/thingsboard/server/common/adaptor/JsonConverterTest.java @@ -1,4 +1,4 @@ -package org.thingsboard.server.common.adaptor; /** + /** * Copyright © 2016-2023 The Thingsboard Authors * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,6 +13,8 @@ package org.thingsboard.server.common.adaptor; /** * See the License for the specific language governing permissions and * limitations under the License. */ +package org.thingsboard.server.common.adaptor; + import com.google.gson.JsonParser; import com.google.gson.JsonSyntaxException; import org.junit.Assert; From 9b51bf7b997e71f676963e91559512c835022a70 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 30 Nov 2023 16:42:31 +0100 Subject: [PATCH 16/17] fixed flaky test --- .../controller/DeviceControllerTest.java | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java index cefe24e935..7cafe2bbcb 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java @@ -28,12 +28,14 @@ import org.junit.Before; import org.junit.Ignore; import org.junit.Test; import org.mockito.AdditionalAnswers; +import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Primary; import org.springframework.test.context.ContextConfiguration; +import org.testcontainers.shaded.org.awaitility.Awaitility; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; @@ -72,6 +74,7 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; import org.thingsboard.server.service.state.DeviceStateService; @@ -83,7 +86,6 @@ import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.containsString; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -1345,27 +1347,27 @@ public class DeviceControllerTest extends AbstractControllerTest { ActionType.ASSIGNED_TO_TENANT, savedDifferentTenant.getId().getId().toString(), savedDifferentTenant.getTitle()); testNotificationUpdateGatewayNever(); - Mockito.verify(deviceStateService, times(1)).onQueueMsg( - argThat(proto -> - proto.getTenantIdMSB() == savedTenant.getUuidId().getMostSignificantBits() && - proto.getTenantIdLSB() == savedTenant.getUuidId().getLeastSignificantBits() && - proto.getDeviceIdMSB() == savedDevice.getUuidId().getMostSignificantBits() && - proto.getDeviceIdLSB() == savedDevice.getUuidId().getLeastSignificantBits() && - proto.getDeleted() - ), - any() - ); - - Mockito.verify(deviceStateService, times(1)).onQueueMsg( - argThat(proto -> - proto.getTenantIdMSB() == savedDifferentTenant.getUuidId().getMostSignificantBits() && - proto.getTenantIdLSB() == savedDifferentTenant.getUuidId().getLeastSignificantBits() && - proto.getDeviceIdMSB() == savedDevice.getUuidId().getMostSignificantBits() && - proto.getDeviceIdLSB() == savedDevice.getUuidId().getLeastSignificantBits() && - proto.getAdded() - ), - any() - ); + ArgumentCaptor protoCaptor = ArgumentCaptor.forClass(TransportProtos.DeviceStateServiceMsgProto.class); + + Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> { + Mockito.verify(deviceStateService, Mockito.atLeastOnce()).onQueueMsg(protoCaptor.capture(), any()); + return protoCaptor.getAllValues().stream().anyMatch(proto -> + proto.getTenantIdMSB() == savedTenant.getUuidId().getMostSignificantBits() && + proto.getTenantIdLSB() == savedTenant.getUuidId().getLeastSignificantBits() && + proto.getDeviceIdMSB() == savedDevice.getUuidId().getMostSignificantBits() && + proto.getDeviceIdLSB() == savedDevice.getUuidId().getLeastSignificantBits() && + proto.getDeleted()); + }); + + Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> { + Mockito.verify(deviceStateService, Mockito.atLeastOnce()).onQueueMsg(protoCaptor.capture(), any()); + return protoCaptor.getAllValues().stream().anyMatch(proto -> + proto.getTenantIdMSB() == savedDifferentTenant.getUuidId().getMostSignificantBits() && + proto.getTenantIdLSB() == savedDifferentTenant.getUuidId().getLeastSignificantBits() && + proto.getDeviceIdMSB() == savedDevice.getUuidId().getMostSignificantBits() && + proto.getDeviceIdLSB() == savedDevice.getUuidId().getLeastSignificantBits() && + proto.getAdded()); + }); login("tenant9@thingsboard.org", "testPassword1"); From 97af50191dcfc3ad05e1d018cd53e91fe6d46c95 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 1 Dec 2023 11:51:05 +0200 Subject: [PATCH 17/17] minor refactoring --- common/proto/pom.xml | 15 --------------- .../server/common/adaptor/JsonConverterTest.java | 2 +- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/common/proto/pom.xml b/common/proto/pom.xml index acaf94cd74..4ccf154ead 100644 --- a/common/proto/pom.xml +++ b/common/proto/pom.xml @@ -52,21 +52,6 @@ com.google.protobuf protobuf-java-util - - io.grpc - grpc-netty-shaded - provided - - - io.grpc - grpc-protobuf - provided - - - io.grpc - grpc-stub - provided - org.springframework.boot spring-boot-starter-web diff --git a/common/proto/src/test/java/org/thingsboard/server/common/adaptor/JsonConverterTest.java b/common/proto/src/test/java/org/thingsboard/server/common/adaptor/JsonConverterTest.java index 9e4bf6d21b..8c32677afb 100644 --- a/common/proto/src/test/java/org/thingsboard/server/common/adaptor/JsonConverterTest.java +++ b/common/proto/src/test/java/org/thingsboard/server/common/adaptor/JsonConverterTest.java @@ -1,4 +1,4 @@ - /** +/** * Copyright © 2016-2023 The Thingsboard Authors * * Licensed under the Apache License, Version 2.0 (the "License");