diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 066c1468fb..f29cc55197 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1493,9 +1493,7 @@ swagger: # Queue configuration parameters queue: - # in-memory or kafka (Apache Kafka). The following queue types are deprecated and will no longer be supported in ThingsBoard 4.0: - # aws-sqs (AWS SQS), pubsub (PubSub), service-bus (Azure Service Bus), rabbitmq (RabbitMQ) - type: "${TB_QUEUE_TYPE:in-memory}" + type: "${TB_QUEUE_TYPE:in-memory}" # in-memory or kafka (Apache Kafka) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). in_memory: stats: @@ -1624,122 +1622,6 @@ queue: print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}" # Time to wait for the stats-loading requests to Kafka to finish kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}" - aws_sqs: - # Use the default credentials provider for AWS SQS - use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" - # Access key ID from AWS IAM user - access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" - # Secret access key from AWS IAM user - secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" - # Region from AWS account - region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" - # Number of threads per each AWS SQS queue in consumer - threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - # Thread pool size for aws_sqs queue producer executor provider. Default value equals to AmazonSQSAsyncClient.DEFAULT_THREAD_POOL_SIZE - producer_thread_pool_size: "${TB_QUEUE_AWS_SQS_EXECUTOR_THREAD_POOL_SIZE:50}" - queue-properties: - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800 - js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - ota-updates: "${TB_QUEUE_AWS_SQS_OTA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - version-control: "${TB_QUEUE_AWS_SQS_VC_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - edge: "${TB_QUEUE_AWS_SQS_EDGE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - pubsub: - # Project ID from Google Cloud - project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" - # API Credentials in JSON format - service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - # Message size for PubSub queue.Value in bytes - max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" - # Number of messages per consumer - max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - # Thread pool size for pubsub queue executor provider. If set to 0 - default pubsub executor provider value will be used (5 * number of available processors) - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:0}" - queue-properties: - # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # Pub/Sub properties for Core subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # Pub/Sub properties for Transport API subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # Pub/Sub properties for Version Control subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # PubSub queue properties - js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # Pub/Sub properties for Transport Api subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - version-control: "${TB_QUEUE_PUBSUB_VC_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # Pub/Sub properties for Edge subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - edge: "${TB_QUEUE_PUBSUB_EDGE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - service_bus: - # Azure namespace - namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" - # Azure Service Bus Shared Access Signatures key name - sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" - # Azure Service Bus Shared Access Signatures key - sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" - # Number of messages per a consumer - max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" - queue-properties: - # Azure Service Bus properties for Rule Engine queues - rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus properties for Core queues - core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus properties for Transport Api queues - transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus properties for Notification queues - notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus queue properties - js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus properties for Version Control queues - version-control: "${TB_QUEUE_SERVICE_BUS_VC_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus properties for Edge queues - edge: "${TB_QUEUE_SERVICE_BUS_EDGE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - rabbitmq: - # By default empty - exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" - # RabbitMQ host used to establish connection - host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" - # RabbitMQ host used to establish a connection - port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" - # Virtual hosts provide logical grouping and separation of resources - virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" - # Username for RabbitMQ user account - username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" - # User password for RabbitMQ user account - password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" - # Network connection between clients and RabbitMQ nodes can fail. RabbitMQ Java client supports automatic recovery of connections and topology (queues, exchanges, bindings, and consumers) - automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" - # The connection timeout for the RabbitMQ connection factory - connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" - # RabbitMQ has a timeout for connection handshake. When clients run in heavily constrained environments, it may be necessary to increase the timeout - handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" - # The maximum number of messages returned in a single call of doPoll() method - max_poll_messages: "${TB_QUEUE_RABBIT_MQ_MAX_POLL_MESSAGES:1}" - queue-properties: - # RabbitMQ properties for Rule Engine queues - rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ properties for Core queues - core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ properties for Transport API queues - transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ properties for Notification queues - notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ queue properties - js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ properties for Version Control queues - version-control: "${TB_QUEUE_RABBIT_MQ_VC_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ properties for Edge queues - edge: "${TB_QUEUE_RABBIT_MQ_EDGE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 transport_api: @@ -1760,7 +1642,7 @@ queue: # Interval in milliseconds to poll api response from transport microservices response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: - # Default topic name of Kafka, RabbitMQ, etc. queue + # Default topic name topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" # Interval in milliseconds to poll messages by Core microservices poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" @@ -1777,7 +1659,7 @@ queue: pack-interval-ms: "${TB_QUEUE_CORE_OTA_PACK_INTERVAL_MS:60000}" # The size of OTA updates notifications fetched from the queue. The queue stores pairs of firmware and device ids pack-size: "${TB_QUEUE_CORE_OTA_PACK_SIZE:100}" - # Stats topic name for queue Kafka, RabbitMQ, etc. + # Stats topic name usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}" stats: # Enable/disable statistics for Core microservices @@ -1808,7 +1690,7 @@ queue: print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}" vc: - # Default topic name for Kafka, RabbitMQ, etc. + # Default topic name topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" # Number of partitions to associate with this queue. Used for scaling the number of messages that can be processed in parallel partitions: "${TB_QUEUE_VC_PARTITIONS:10}" @@ -1818,7 +1700,7 @@ queue: pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:180000}" # Timeout for a request to VC-executor (for a request for the version of the entity, for a commit charge, etc.) request-timeout: "${TB_QUEUE_VC_REQUEST_TIMEOUT:180000}" - # Queue settings for Kafka, RabbitMQ, etc. Limit for single message size + # Limit for single queue message size msg-chunk-size: "${TB_QUEUE_VC_MSG_CHUNK_SIZE:250000}" js: # JS Eval request topic @@ -1859,7 +1741,7 @@ queue: # Interval in milliseconds to poll messages poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" edge: - # Default topic name for Kafka, RabbitMQ, etc. + # Default topic name topic: "${TB_QUEUE_EDGE_TOPIC:tb_edge}" # Amount of partitions used by Edge services partitions: "${TB_QUEUE_EDGE_PARTITIONS:10}" diff --git a/common/queue/pom.xml b/common/queue/pom.xml index 6fad5efd5a..44e2239e05 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -64,18 +64,10 @@ org.apache.kafka kafka-clients - - com.amazonaws - aws-java-sdk-sqs - com.google.cloud google-cloud-pubsub - - com.microsoft.azure - azure-servicebus - com.rabbitmq amqp-client diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/RuleEngineTbQueueAdminFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/RuleEngineTbQueueAdminFactory.java index 87ad934c63..057351a449 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/RuleEngineTbQueueAdminFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/RuleEngineTbQueueAdminFactory.java @@ -19,21 +19,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; -import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; -import org.thingsboard.server.queue.pubsub.TbPubSubSettings; -import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; -import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; -import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; -import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; @Configuration public class RuleEngineTbQueueAdminFactory { @@ -43,56 +31,12 @@ public class RuleEngineTbQueueAdminFactory { @Autowired(required = false) private TbKafkaSettings kafkaSettings; - @Autowired(required = false) - private TbAwsSqsQueueAttributes awsSqsQueueAttributes; - @Autowired(required = false) - private TbAwsSqsSettings awsSqsSettings; - - @Autowired(required = false) - private TbPubSubSubscriptionSettings pubSubSubscriptionSettings; - @Autowired(required = false) - private TbPubSubSettings pubSubSettings; - - @Autowired(required = false) - private TbRabbitMqQueueArguments rabbitMqQueueArguments; - @Autowired(required = false) - private TbRabbitMqSettings rabbitMqSettings; - - @Autowired(required = false) - private TbServiceBusQueueConfigs serviceBusQueueConfigs; - @Autowired(required = false) - private TbServiceBusSettings serviceBusSettings; - @ConditionalOnExpression("'${queue.type:null}'=='kafka'") @Bean public TbQueueAdmin createKafkaAdmin() { return new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); } - @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs'") - @Bean - public TbQueueAdmin createAwsSqsAdmin() { - return new TbAwsSqsAdmin(awsSqsSettings, awsSqsQueueAttributes.getRuleEngineAttributes()); - } - - @ConditionalOnExpression("'${queue.type:null}'=='pubsub'") - @Bean - public TbQueueAdmin createPubSubAdmin() { - return new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); - } - - @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'") - @Bean - public TbQueueAdmin createRabbitMqAdmin() { - return new TbRabbitMqAdmin(rabbitMqSettings, rabbitMqQueueArguments.getRuleEngineArgs()); - } - - @ConditionalOnExpression("'${queue.type:null}'=='service-bus'") - @Bean - public TbQueueAdmin createServiceBusAdmin() { - return new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); - } - @ConditionalOnExpression("'${queue.type:null}'=='in-memory'") @Bean public TbQueueAdmin createInMemoryAdmin() { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java deleted file mode 100644 index 3f920778e6..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java +++ /dev/null @@ -1,137 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.azure.servicebus; - -import com.microsoft.azure.servicebus.management.ManagementClient; -import com.microsoft.azure.servicebus.management.QueueDescription; -import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; -import com.microsoft.azure.servicebus.primitives.MessagingEntityAlreadyExistsException; -import com.microsoft.azure.servicebus.primitives.ServiceBusException; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.util.PropertyUtils; - -import java.io.IOException; -import java.time.Duration; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - -@Slf4j -@Deprecated(forRemoval = true, since = "3.9") // for removal in 4.0 -public class TbServiceBusAdmin implements TbQueueAdmin { - private final String MAX_SIZE = "maxSizeInMb"; - private final String MESSAGE_TIME_TO_LIVE = "messageTimeToLiveInSec"; - private final String LOCK_DURATION = "lockDurationInSec"; - - private final Map queueConfigs; - private final Set queues = ConcurrentHashMap.newKeySet(); - - private final ManagementClient client; - - public TbServiceBusAdmin(TbServiceBusSettings serviceBusSettings, Map queueConfigs) { - this.queueConfigs = queueConfigs; - - ConnectionStringBuilder builder = new ConnectionStringBuilder( - serviceBusSettings.getNamespaceName(), - "queues", - serviceBusSettings.getSasKeyName(), - serviceBusSettings.getSasKey()); - - client = new ManagementClient(builder); - - try { - client.getQueues().forEach(queueDescription -> queues.add(queueDescription.getPath())); - } catch (ServiceBusException | InterruptedException e) { - log.error("Failed to get queues.", e); - throw new RuntimeException("Failed to get queues.", e); - } - } - - @Override - public void createTopicIfNotExists(String topic, String properties) { - if (queues.contains(topic)) { - return; - } - - try { - QueueDescription queueDescription = new QueueDescription(topic); - queueDescription.setRequiresDuplicateDetection(false); - setQueueConfigs(queueDescription, PropertyUtils.getProps(queueConfigs, properties)); - - client.createQueue(queueDescription); - queues.add(topic); - } catch (ServiceBusException | InterruptedException e) { - if (e instanceof MessagingEntityAlreadyExistsException) { - queues.add(topic); - log.info("[{}] queue already exists.", topic); - } else { - log.error("Failed to create queue: [{}]", topic, e); - } - } - } - - @Override - public void deleteTopic(String topic) { - if (queues.contains(topic)) { - doDelete(topic); - } else { - try { - if (client.getQueue(topic) != null) { - doDelete(topic); - } else { - log.warn("Azure Service Bus Queue [{}] is not exist.", topic); - } - } catch (ServiceBusException | InterruptedException e) { - log.error("Failed to delete Azure Service Bus queue [{}]", topic, e); - } - } - } - - private void doDelete(String topic) { - try { - client.deleteTopic(topic); - } catch (ServiceBusException | InterruptedException e) { - log.error("Failed to delete Azure Service Bus queue [{}]", topic, e); - } - } - - private void setQueueConfigs(QueueDescription queueDescription, Map queueConfigs) { - queueConfigs.forEach((confKey, confValue) -> { - switch (confKey) { - case MAX_SIZE: - queueDescription.setMaxSizeInMB(Long.parseLong(confValue)); - break; - case MESSAGE_TIME_TO_LIVE: - queueDescription.setDefaultMessageTimeToLive(Duration.ofSeconds(Long.parseLong(confValue))); - break; - case LOCK_DURATION: - queueDescription.setLockDuration(Duration.ofSeconds(Long.parseLong(confValue))); - break; - default: - log.error("Unknown config: [{}]", confKey); - } - }); - } - - public void destroy() { - try { - client.close(); - } catch (IOException e) { - log.error("Failed to close ManagementClient."); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java deleted file mode 100644 index 8e2b32aab5..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java +++ /dev/null @@ -1,175 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.azure.servicebus; - -import com.google.gson.Gson; -import com.google.protobuf.InvalidProtocolBufferException; -import com.microsoft.azure.servicebus.TransactionContext; -import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; -import com.microsoft.azure.servicebus.primitives.CoreMessageReceiver; -import com.microsoft.azure.servicebus.primitives.MessageWithDeliveryTag; -import com.microsoft.azure.servicebus.primitives.MessagingEntityType; -import com.microsoft.azure.servicebus.primitives.MessagingFactory; -import com.microsoft.azure.servicebus.primitives.SettleModePair; -import lombok.extern.slf4j.Slf4j; -import org.apache.qpid.proton.amqp.messaging.Data; -import org.apache.qpid.proton.amqp.transport.ReceiverSettleMode; -import org.apache.qpid.proton.amqp.transport.SenderSettleMode; -import org.springframework.util.CollectionUtils; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueMsgDecoder; -import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.time.Duration; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -@Slf4j -public class TbServiceBusConsumerTemplate extends AbstractTbQueueConsumerTemplate { - private final TbQueueAdmin admin; - private final TbQueueMsgDecoder decoder; - private final TbServiceBusSettings serviceBusSettings; - - private final Gson gson = new Gson(); - - private Set receivers; - private final Map> pendingMessages = new ConcurrentHashMap<>(); - private volatile int messagesPerQueue; - - public TbServiceBusConsumerTemplate(TbQueueAdmin admin, TbServiceBusSettings serviceBusSettings, String topic, TbQueueMsgDecoder decoder) { - super(topic); - this.admin = admin; - this.decoder = decoder; - this.serviceBusSettings = serviceBusSettings; - } - - @Override - protected List doPoll(long durationInMillis) { - List>> messageFutures = - receivers.stream() - .map(receiver -> receiver - .receiveAsync(messagesPerQueue, Duration.ofMillis(durationInMillis)) - .whenComplete((messages, err) -> { - if (!CollectionUtils.isEmpty(messages)) { - pendingMessages.put(receiver, messages); - } else if (err != null) { - log.error("Failed to receive messages.", err); - } - })) - .collect(Collectors.toList()); - try { - return fromList(messageFutures) - .get() - .stream() - .flatMap(messages -> CollectionUtils.isEmpty(messages) ? Stream.empty() : messages.stream()) - .collect(Collectors.toList()); - } catch (InterruptedException | ExecutionException e) { - if (stopped) { - log.info("[{}] Service Bus consumer is stopped.", getTopic()); - } else { - log.error("Failed to receive messages", e); - } - return Collections.emptyList(); - } - } - - @Override - protected void doSubscribe(List topicNames) { - createReceivers(); - messagesPerQueue = receivers.size() / Math.max(partitions.size(), 1); - } - - @Override - protected void doCommit() { - pendingMessages.forEach((receiver, msgs) -> - msgs.forEach(msg -> receiver.completeMessageAsync(msg.getDeliveryTag(), TransactionContext.NULL_TXN))); - pendingMessages.clear(); - } - - @Override - protected void doUnsubscribe() { - receivers.forEach(CoreMessageReceiver::closeAsync); - } - - private void createReceivers() { - List> receiverFutures = partitions.stream() - .map(TopicPartitionInfo::getFullTopicName) - .map(queue -> { - MessagingFactory factory; - try { - factory = MessagingFactory.createFromConnectionStringBuilder(createConnection(queue)); - } catch (InterruptedException | ExecutionException e) { - log.error("Failed to create factory for the queue [{}]", queue); - throw new RuntimeException("Failed to create the factory", e); - } - - return CoreMessageReceiver.create(factory, queue, queue, 0, - new SettleModePair(SenderSettleMode.UNSETTLED, ReceiverSettleMode.SECOND), - MessagingEntityType.QUEUE); - }).collect(Collectors.toList()); - - try { - receivers = new HashSet<>(fromList(receiverFutures).get()); - } catch (InterruptedException | ExecutionException e) { - if (stopped) { - log.info("[{}] Service Bus consumer is stopped.", getTopic()); - } else { - log.error("Failed to create receivers", e); - } - } - } - - private ConnectionStringBuilder createConnection(String queue) { - admin.createTopicIfNotExists(queue); - return new ConnectionStringBuilder( - serviceBusSettings.getNamespaceName(), - queue, - serviceBusSettings.getSasKeyName(), - serviceBusSettings.getSasKey()); - } - - private CompletableFuture> fromList(List> futures) { - @SuppressWarnings("unchecked") - CompletableFuture>[] arrayFuture = new CompletableFuture[futures.size()]; - futures.toArray(arrayFuture); - - return CompletableFuture - .allOf(arrayFuture) - .thenApply(v -> futures - .stream() - .map(CompletableFuture::join) - .collect(Collectors.toList())); - } - - @Override - protected T decode(MessageWithDeliveryTag data) throws InvalidProtocolBufferException { - DefaultTbQueueMsg msg = gson.fromJson(new String(((Data) data.getMessage().getBody()).getValue().getArray()), DefaultTbQueueMsg.class); - return decoder.decode(msg); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java deleted file mode 100644 index 119f28079b..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusProducerTemplate.java +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.azure.servicebus; - -import com.google.gson.Gson; -import com.microsoft.azure.servicebus.IMessage; -import com.microsoft.azure.servicebus.Message; -import com.microsoft.azure.servicebus.QueueClient; -import com.microsoft.azure.servicebus.ReceiveMode; -import com.microsoft.azure.servicebus.primitives.ConnectionStringBuilder; -import com.microsoft.azure.servicebus.primitives.ServiceBusException; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -@Slf4j -public class TbServiceBusProducerTemplate implements TbQueueProducer { - private final String defaultTopic; - private final Gson gson = new Gson(); - private final TbQueueAdmin admin; - private final TbServiceBusSettings serviceBusSettings; - private final Map clients = new ConcurrentHashMap<>(); - private final ExecutorService executorService; - - public TbServiceBusProducerTemplate(TbQueueAdmin admin, TbServiceBusSettings serviceBusSettings, String defaultTopic) { - this.admin = admin; - this.defaultTopic = defaultTopic; - this.serviceBusSettings = serviceBusSettings; - executorService = Executors.newCachedThreadPool(); - } - - @Override - public void init() { - - } - - @Override - public String getDefaultTopic() { - return defaultTopic; - } - - @Override - public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { - IMessage message = new Message(gson.toJson(new DefaultTbQueueMsg(msg))); - CompletableFuture future = getClient(tpi.getFullTopicName()).sendAsync(message); - future.whenCompleteAsync((success, err) -> { - if (err != null) { - callback.onFailure(err); - } else { - callback.onSuccess(null); - } - }, executorService); - } - - @Override - public void stop() { - clients.forEach((t, client) -> { - try { - client.close(); - } catch (ServiceBusException e) { - log.error("Failed to close QueueClient.", e); - } - }); - - if (executorService != null) { - executorService.shutdownNow(); - } - } - - private QueueClient getClient(String topic) { - return clients.computeIfAbsent(topic, k -> { - admin.createTopicIfNotExists(topic); - ConnectionStringBuilder builder = - new ConnectionStringBuilder( - serviceBusSettings.getNamespaceName(), - topic, - serviceBusSettings.getSasKeyName(), - serviceBusSettings.getSasKey()); - try { - return new QueueClient(builder, ReceiveMode.PEEKLOCK); - } catch (InterruptedException | ServiceBusException e) { - log.error("Failed to create new client for the Queue: [{}]", topic, e); - throw new RuntimeException("Failed to create new client for the Queue", e); - } - }); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java deleted file mode 100644 index 80da7846ea..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.azure.servicebus; - -import jakarta.annotation.PostConstruct; -import lombok.Getter; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.PropertyUtils; - -import java.util.Map; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='service-bus'") -public class TbServiceBusQueueConfigs { - - @Value("${queue.service-bus.queue-properties.core:}") - private String coreProperties; - @Value("${queue.service-bus.queue-properties.rule-engine:}") - private String ruleEngineProperties; - @Value("${queue.service-bus.queue-properties.transport-api:}") - private String transportApiProperties; - @Value("${queue.service-bus.queue-properties.notifications:}") - private String notificationsProperties; - @Value("${queue.service-bus.queue-properties.js-executor:}") - private String jsExecutorProperties; - @Value("${queue.service-bus.queue-properties.version-control:}") - private String vcProperties; - @Value("${queue.service-bus.queue-properties.edge:}") - private String edgeProperties; - - @Getter - private Map coreConfigs; - @Getter - private Map ruleEngineConfigs; - @Getter - private Map transportApiConfigs; - @Getter - private Map notificationsConfigs; - @Getter - private Map jsExecutorConfigs; - @Getter - private Map vcConfigs; - @Getter - private Map edgeConfigs; - - @PostConstruct - private void init() { - coreConfigs = PropertyUtils.getProps(coreProperties); - ruleEngineConfigs = PropertyUtils.getProps(ruleEngineProperties); - transportApiConfigs = PropertyUtils.getProps(transportApiProperties); - notificationsConfigs = PropertyUtils.getProps(notificationsProperties); - jsExecutorConfigs = PropertyUtils.getProps(jsExecutorProperties); - vcConfigs = PropertyUtils.getProps(vcProperties); - edgeConfigs = PropertyUtils.getProps(edgeProperties); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java deleted file mode 100644 index 98bb11a839..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusSettings.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.azure.servicebus; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; - -@Slf4j -@ConditionalOnExpression("'${queue.type:null}'=='service-bus'") -@Component -@Data -public class TbServiceBusSettings { - @Value("${queue.service_bus.namespace_name}") - private String namespaceName; - @Value("${queue.service_bus.sas_key_name}") - private String sasKeyName; - @Value("${queue.service_bus.sas_key}") - private String sasKey; - @Value("${queue.service_bus.max_messages}") - private int maxMessages; -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java deleted file mode 100644 index 310cc1d1c5..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ /dev/null @@ -1,317 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import com.google.protobuf.util.JsonFormat; -import jakarta.annotation.PreDestroy; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.queue.Queue; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; -import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; -import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; -import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; -import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; - -import java.nio.charset.StandardCharsets; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='monolith'") -public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { - - private final TopicService topicService; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbAwsSqsSettings sqsSettings; - private final TbQueueVersionControlSettings vcSettings; - private final TbQueueEdgeSettings edgeSettings; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - private final TbQueueAdmin otaAdmin; - private final TbQueueAdmin vcAdmin; - private final TbQueueAdmin edgeAdmin; - - public AwsSqsMonolithQueueFactory(TopicService topicService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbAwsSqsSettings sqsSettings, - TbQueueVersionControlSettings vcSettings, - TbQueueEdgeSettings edgeSettings, - TbAwsSqsQueueAttributes sqsQueueAttributes, - TbQueueRemoteJsInvokeSettings jsInvokeSettings) { - this.topicService = topicService; - this.coreSettings = coreSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.sqsSettings = sqsSettings; - this.vcSettings = vcSettings; - this.edgeSettings = edgeSettings; - this.jsInvokeSettings = jsInvokeSettings; - - this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); - this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); - this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); - this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); - this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); - this.otaAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getOtaAttributes()); - this.vcAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getVcAttributes()); - this.edgeAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getEdgeAttributes()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, - topicService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @Override - public TbQueueConsumer> createToVersionControlMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(vcAdmin, sqsSettings, topicService.buildTopicName(vcSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) - ); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { - return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, topicService.buildTopicName(configuration.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, - topicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, - topicService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, topicService.buildTopicName(transportApiSettings.getRequestsTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, topicService.buildTopicName(transportApiSettings.getResponsesTopic())); - } - - @Override - @Bean - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbAwsSqsProducerTemplate<>(jsExecutorAdmin, sqsSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbAwsSqsConsumerTemplate<>(jsExecutorAdmin, sqsSettings, - jsInvokeSettings.getResponseTopic() + "_" + serviceInfoProvider.getServiceId(), - msg -> { - RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); - } - - @Override - public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); - } - - @Override - public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToOtaPackageStateServiceMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(otaAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToOtaPackageStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { - return new TbAwsSqsProducerTemplate<>(otaAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); - } - - @Override - public TbQueueProducer> createVersionControlMsgProducer() { - return new TbAwsSqsProducerTemplate<>(vcAdmin, sqsSettings, topicService.buildTopicName(vcSettings.getTopic())); - } - - public TbQueueProducer> createHousekeeperMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); - } - - @Override - public TbQueueConsumer> createHousekeeperMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); - } - - @Override - public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createEdgeMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(edgeAdmin, sqsSettings, topicService.buildTopicName(edgeSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createEdgeMsgProducer() { - return new TbAwsSqsProducerTemplate<>(edgeAdmin, sqsSettings, topicService.buildTopicName(edgeSettings.getTopic())); - } - - @Override - public TbQueueConsumer> createToEdgeNotificationsMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, - topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createEdgeNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, - topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @Override - public TbQueueProducer> createEdgeEventMsgProducer() { - return null; - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - if (otaAdmin != null) { - otaAdmin.destroy(); - } - if (vcAdmin != null) { - vcAdmin.destroy(); - } - if (edgeAdmin != null) { - edgeAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java deleted file mode 100644 index 515c1bba44..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ /dev/null @@ -1,291 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import com.google.protobuf.util.JsonFormat; -import jakarta.annotation.PreDestroy; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; -import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; -import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; -import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; - -import java.nio.charset.StandardCharsets; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-core'") -public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { - - private final TbAwsSqsSettings sqsSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueCoreSettings coreSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TopicService topicService; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbQueueVersionControlSettings vcSettings; - private final TbQueueEdgeSettings edgeSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - private final TbQueueAdmin otaAdmin; - private final TbQueueAdmin vcAdmin; - private final TbQueueAdmin edgeAdmin; - - public AwsSqsTbCoreQueueFactory(TbAwsSqsSettings sqsSettings, - TbQueueCoreSettings coreSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TopicService topicService, - TbQueueVersionControlSettings vcSettings, - TbQueueEdgeSettings edgeSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, - TbAwsSqsQueueAttributes sqsQueueAttributes, - TbQueueTransportNotificationSettings transportNotificationSettings) { - this.sqsSettings = sqsSettings; - this.coreSettings = coreSettings; - this.transportApiSettings = transportApiSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.edgeSettings = edgeSettings; - this.topicService = topicService; - this.serviceInfoProvider = serviceInfoProvider; - this.jsInvokeSettings = jsInvokeSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.vcSettings = vcSettings; - - this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); - this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); - this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); - this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); - this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); - this.otaAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getOtaAttributes()); - this.vcAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getVcAttributes()); - this.edgeAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getEdgeAttributes()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, - topicService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, - topicService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, topicService.buildTopicName(transportApiSettings.getRequestsTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, topicService.buildTopicName(transportApiSettings.getResponsesTopic())); - } - - @Override - @Bean - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbAwsSqsProducerTemplate<>(jsExecutorAdmin, sqsSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbAwsSqsConsumerTemplate<>(jsExecutorAdmin, sqsSettings, - jsInvokeSettings.getResponseTopic() + "_" + serviceInfoProvider.getServiceId(), - msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); - } - - @Override - public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); - } - - @Override - public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToOtaPackageStateServiceMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(otaAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToOtaPackageStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { - return new TbAwsSqsProducerTemplate<>(otaAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); - } - - @Override - public TbQueueProducer> createVersionControlMsgProducer() { - return new TbAwsSqsProducerTemplate<>(vcAdmin, sqsSettings, topicService.buildTopicName(vcSettings.getTopic())); - } - - @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); - } - - @Override - public TbQueueConsumer> createHousekeeperMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); - } - - @Override - public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createEdgeMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(edgeAdmin, sqsSettings, topicService.buildTopicName(edgeSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createEdgeMsgProducer() { - return new TbAwsSqsProducerTemplate<>(edgeAdmin, sqsSettings, topicService.buildTopicName(edgeSettings.getTopic())); - } - - @Override - public TbQueueConsumer> createToEdgeNotificationsMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, - topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createEdgeNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, - topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - if (otaAdmin != null) { - otaAdmin.destroy(); - } - if (vcAdmin != null) { - vcAdmin.destroy(); - } - if (edgeAdmin != null) { - edgeAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java deleted file mode 100644 index a93aba2764..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java +++ /dev/null @@ -1,208 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import com.google.protobuf.util.JsonFormat; -import jakarta.annotation.PreDestroy; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.queue.Queue; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; -import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; -import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; - -import java.nio.charset.StandardCharsets; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-rule-engine'") -public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { - - private final TopicService topicService; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbAwsSqsSettings sqsSettings; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbQueueEdgeSettings edgeSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin notificationAdmin; - private final TbQueueAdmin otaAdmin; - private final TbQueueAdmin edgeAdmin; - - public AwsSqsTbRuleEngineQueueFactory(TopicService topicService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbAwsSqsSettings sqsSettings, - TbAwsSqsQueueAttributes sqsQueueAttributes, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbQueueEdgeSettings edgeSettings) { - this.topicService = topicService; - this.coreSettings = coreSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.sqsSettings = sqsSettings; - this.jsInvokeSettings = jsInvokeSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.edgeSettings = edgeSettings; - - this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); - this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); - this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); - this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); - this.otaAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getOtaAttributes()); - this.edgeAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getEdgeAttributes()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createEdgeMsgProducer() { - return new TbAwsSqsProducerTemplate<>(edgeAdmin, sqsSettings, topicService.buildTopicName(edgeSettings.getTopic())); - } - - @Override - public TbQueueProducer> createEdgeNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { - return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, topicService.buildTopicName(configuration.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, - topicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - @Bean - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbAwsSqsProducerTemplate<>(jsExecutorAdmin, sqsSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbAwsSqsConsumerTemplate<>(jsExecutorAdmin, sqsSettings, - jsInvokeSettings.getResponseTopic() + "_" + serviceInfoProvider.getServiceId(), - msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); - } - - @Override - public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); - } - - @Override - public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { - return new TbAwsSqsProducerTemplate<>(otaAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); - } - - @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - if (otaAdmin != null) { - otaAdmin.destroy(); - } - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbVersionControlQueueFactory.java deleted file mode 100644 index 67ff0393bd..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbVersionControlQueueFactory.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import jakarta.annotation.PreDestroy; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; -import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; -import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; -import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-vc-executor'") -public class AwsSqsTbVersionControlQueueFactory implements TbVersionControlQueueFactory { - - private final TbAwsSqsSettings sqsSettings; - private final TbQueueCoreSettings coreSettings; - private final TbQueueVersionControlSettings vcSettings; - private final TopicService topicService; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin notificationAdmin; - private final TbQueueAdmin vcAdmin; - - public AwsSqsTbVersionControlQueueFactory(TbAwsSqsSettings sqsSettings, - TbQueueCoreSettings coreSettings, - TbQueueVersionControlSettings vcSettings, - TbAwsSqsQueueAttributes sqsQueueAttributes, - TopicService topicService - ) { - this.sqsSettings = sqsSettings; - this.coreSettings = coreSettings; - this.vcSettings = vcSettings; - this.topicService = topicService; - - this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); - this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); - this.vcAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getVcAttributes()); - } - - @Override - public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueConsumer> createToVersionControlMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(vcAdmin, sqsSettings, topicService.buildTopicName(vcSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) - ); - } - - @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - if (vcAdmin != null) { - vcAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java deleted file mode 100644 index 407ef86a99..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java +++ /dev/null @@ -1,153 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import jakarta.annotation.PreDestroy; -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; -import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; -import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; -import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && (('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true') || '${service.type:null}'=='tb-transport')") -@Slf4j -public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbAwsSqsSettings sqsSettings; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TopicService topicService; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - private final TbQueueAdmin ruleEngineAdmin; - - public AwsSqsTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbAwsSqsSettings sqsSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbAwsSqsQueueAttributes sqsQueueAttributes, - TbQueueRuleEngineSettings ruleEngineSettings, - TopicService topicService) { - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.sqsSettings = sqsSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.coreSettings = coreSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.topicService = topicService; - - this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); - this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); - this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); - this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbQueueProducer> producerTemplate = - new TbAwsSqsProducerTemplate<>(transportApiAdmin, sqsSettings, topicService.buildTopicName(transportApiSettings.getRequestsTopic())); - - TbQueueConsumer> consumerTemplate = - new TbAwsSqsConsumerTemplate<>(transportApiAdmin, sqsSettings, - topicService.buildTopicName(transportApiSettings.getResponsesTopic() + "_" + serviceInfoProvider.getServiceId()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(transportApiAdmin); - templateBuilder.requestTemplate(producerTemplate); - templateBuilder.responseTemplate(consumerTemplate); - templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); - templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); - templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); - return templateBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbAwsSqsProducerTemplate<>(ruleEngineAdmin, sqsSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueConsumer> createTransportNotificationsConsumer() { - return new TbAwsSqsConsumerTemplate<>(notificationAdmin, sqsSettings, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic() + "_" + serviceInfoProvider.getServiceId()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); - } - - @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java deleted file mode 100644 index 05bef819b7..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ /dev/null @@ -1,315 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import com.google.protobuf.util.JsonFormat; -import jakarta.annotation.PreDestroy; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.queue.Queue; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; -import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; -import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubSettings; -import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; - -import java.nio.charset.StandardCharsets; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='monolith'") -public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { - - private final TbPubSubSettings pubSubSettings; - private final TbQueueCoreSettings coreSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TopicService topicService; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; - private final TbQueueVersionControlSettings vcSettings; - private final TbQueueEdgeSettings edgeSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - private final TbQueueAdmin vcAdmin; - private final TbQueueAdmin edgeAdmin; - - public PubSubMonolithQueueFactory(TbPubSubSettings pubSubSettings, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TopicService topicService, - TbServiceInfoProvider serviceInfoProvider, - TbPubSubSubscriptionSettings pubSubSubscriptionSettings, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, - TbQueueVersionControlSettings vcSettings, - TbQueueEdgeSettings edgeSettings) { - this.pubSubSettings = pubSubSettings; - this.coreSettings = coreSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.topicService = topicService; - this.serviceInfoProvider = serviceInfoProvider; - this.vcSettings = vcSettings; - this.edgeSettings = edgeSettings; - - this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); - this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); - this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); - this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); - this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); - this.vcAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getVcSettings()); - this.edgeAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getEdgeSettings()); - - this.jsInvokeSettings = jsInvokeSettings; - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueConsumer> createToVersionControlMsgConsumer() { - return new TbPubSubConsumerTemplate<>(vcAdmin, pubSubSettings, topicService.buildTopicName(vcSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) - ); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { - return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, topicService.buildTopicName(configuration.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, - topicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, - topicService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, topicService.buildTopicName(transportApiSettings.getRequestsTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, topicService.buildTopicName(transportApiSettings.getResponsesTopic())); - } - - @Override - @Bean - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(jsExecutorAdmin, pubSubSettings, - jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> { - RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); - } - - @Override - public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { - return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToOtaPackageStateServiceMsgConsumer() { - return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToOtaPackageStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); - } - - @Override - public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); - } - - @Override - public TbQueueProducer> createVersionControlMsgProducer() { - return new TbPubSubProducerTemplate<>(vcAdmin, pubSubSettings, topicService.buildTopicName(vcSettings.getTopic())); - } - - @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); - } - - @Override - public TbQueueConsumer> createHousekeeperMsgConsumer() { - return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); - } - - @Override - public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { - return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createEdgeMsgProducer() { - return new TbPubSubProducerTemplate<>(edgeAdmin, pubSubSettings, topicService.buildTopicName(edgeSettings.getTopic())); - } - - @Override - public TbQueueConsumer> createEdgeMsgConsumer() { - return new TbPubSubConsumerTemplate<>(edgeAdmin, pubSubSettings, topicService.buildTopicName(edgeSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToEdgeNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, - topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createEdgeNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, - topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @Override - public TbQueueProducer> createEdgeEventMsgProducer() { - return null; - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - if (vcAdmin != null) { - vcAdmin.destroy(); - } - if (edgeAdmin != null) { - edgeAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java deleted file mode 100644 index 919d895a97..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ /dev/null @@ -1,278 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import com.google.protobuf.util.JsonFormat; -import jakarta.annotation.PreDestroy; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; -import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubSettings; -import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; - -import java.nio.charset.StandardCharsets; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-core'") -public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { - - private final TbPubSubSettings pubSubSettings; - private final TbQueueCoreSettings coreSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TopicService topicService; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueEdgeSettings edgeSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin edgeAdmin; - - public PubSubTbCoreQueueFactory(TbPubSubSettings pubSubSettings, - TbQueueCoreSettings coreSettings, - TbQueueTransportApiSettings transportApiSettings, - TopicService topicService, - TbServiceInfoProvider serviceInfoProvider, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueEdgeSettings edgeSettings, - TbPubSubSubscriptionSettings pubSubSubscriptionSettings) { - this.pubSubSettings = pubSubSettings; - this.coreSettings = coreSettings; - this.transportApiSettings = transportApiSettings; - this.topicService = topicService; - this.serviceInfoProvider = serviceInfoProvider; - this.jsInvokeSettings = jsInvokeSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.edgeSettings = edgeSettings; - - this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); - this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); - this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); - this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); - this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); - this.edgeAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getEdgeSettings()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, - topicService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, - topicService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, topicService.buildTopicName(transportApiSettings.getRequestsTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, topicService.buildTopicName(transportApiSettings.getResponsesTopic())); - } - - @Override - @Bean - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(jsExecutorAdmin, pubSubSettings, - jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); - } - - @Override - public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { - return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToOtaPackageStateServiceMsgConsumer() { - return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToOtaPackageStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); - } - - @Override - public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); - } - - @Override - public TbQueueProducer> createVersionControlMsgProducer() { - //TODO: version-control - return null; - } - - @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); - } - - @Override - public TbQueueConsumer> createHousekeeperMsgConsumer() { - return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); - } - - @Override - public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { - return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createEdgeMsgConsumer() { - return new TbPubSubConsumerTemplate<>(edgeAdmin, pubSubSettings, topicService.buildTopicName(edgeSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createEdgeMsgProducer() { - return new TbPubSubProducerTemplate<>(edgeAdmin, pubSubSettings, topicService.buildTopicName(edgeSettings.getTopic())); - } - - @Override - public TbQueueConsumer> createToEdgeNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, - topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createEdgeNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, - topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (edgeAdmin != null) { - edgeAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java deleted file mode 100644 index 671da15f20..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ /dev/null @@ -1,209 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import com.google.protobuf.util.JsonFormat; -import jakarta.annotation.PreDestroy; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.queue.Queue; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; -import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubSettings; -import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; - -import java.nio.charset.StandardCharsets; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-rule-engine'") -public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { - - private final TbPubSubSettings pubSubSettings; - private final TbQueueCoreSettings coreSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TopicService topicService; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbQueueEdgeSettings edgeSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin notificationAdmin; - private final TbQueueAdmin edgeAdmin; - - public PubSubTbRuleEngineQueueFactory(TbPubSubSettings pubSubSettings, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TopicService topicService, - TbServiceInfoProvider serviceInfoProvider, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbPubSubSubscriptionSettings pubSubSubscriptionSettings, - TbQueueEdgeSettings edgeSettings) { - this.pubSubSettings = pubSubSettings; - this.coreSettings = coreSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.topicService = topicService; - this.serviceInfoProvider = serviceInfoProvider; - this.jsInvokeSettings = jsInvokeSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.edgeSettings = edgeSettings; - - this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); - this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); - this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); - this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); - this.edgeAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getEdgeSettings()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createEdgeMsgProducer() { - return new TbPubSubProducerTemplate<>(edgeAdmin, pubSubSettings, topicService.buildTopicName(edgeSettings.getTopic())); - } - - @Override - public TbQueueProducer> createEdgeNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @Override - public TbQueueProducer> createEdgeEventMsgProducer() { - return null; - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { - return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, topicService.buildTopicName(configuration.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, - topicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - @Bean - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbPubSubProducerTemplate<>(jsExecutorAdmin, pubSubSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(jsExecutorAdmin, pubSubSettings, - jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); - } - - @Override - public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); - } - - @Override - public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); - } - - @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java deleted file mode 100644 index b6da4e2973..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import jakarta.annotation.PreDestroy; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; -import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubSettings; -import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-vc-executor'") -public class PubSubTbVersionControlQueueFactory implements TbVersionControlQueueFactory { - - private final TbPubSubSettings pubSubSettings; - private final TbQueueCoreSettings coreSettings; - private final TbQueueVersionControlSettings vcSettings; - private final TopicService topicService; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin notificationAdmin; - private final TbQueueAdmin vcAdmin; - - public PubSubTbVersionControlQueueFactory(TbPubSubSettings pubSubSettings, - TbQueueCoreSettings coreSettings, - TbQueueVersionControlSettings vcSettings, - TbPubSubSubscriptionSettings pubSubSubscriptionSettings, - TopicService topicService - ) { - this.pubSubSettings = pubSubSettings; - this.coreSettings = coreSettings; - this.vcSettings = vcSettings; - this.topicService = topicService; - - this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); - this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); - this.vcAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getVcSettings()); - } - - @Override - public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueConsumer> createToVersionControlMsgConsumer() { - return new TbPubSubConsumerTemplate<>(vcAdmin, pubSubSettings, topicService.buildTopicName(vcSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) - ); - } - - @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - if (vcAdmin != null) { - vcAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java deleted file mode 100644 index a7500d2083..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java +++ /dev/null @@ -1,153 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import jakarta.annotation.PreDestroy; -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; -import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; -import org.thingsboard.server.queue.pubsub.TbPubSubSettings; -import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && (('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true') || '${service.type:null}'=='tb-transport')") -@Slf4j -public class PubSubTransportQueueFactory implements TbTransportQueueFactory { - - private final TbPubSubSettings pubSubSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueCoreSettings coreSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TopicService topicService; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public PubSubTransportQueueFactory(TbPubSubSettings pubSubSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbPubSubSubscriptionSettings pubSubSubscriptionSettings, - TopicService topicService) { - this.pubSubSettings = pubSubSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.coreSettings = coreSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.topicService = topicService; - - this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); - this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); - this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); - this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbQueueProducer> producer = new TbPubSubProducerTemplate<>(transportApiAdmin, pubSubSettings, topicService.buildTopicName(transportApiSettings.getRequestsTopic())); - TbQueueConsumer> consumer = new TbPubSubConsumerTemplate<>(transportApiAdmin, pubSubSettings, - topicService.buildTopicName(transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(transportApiAdmin); - templateBuilder.requestTemplate(producer); - templateBuilder.responseTemplate(consumer); - templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); - templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); - templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); - return templateBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbPubSubProducerTemplate<>(ruleEngineAdmin, pubSubSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueConsumer> createTransportNotificationsConsumer() { - return new TbPubSubConsumerTemplate<>(notificationAdmin, pubSubSettings, - topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); - } - - @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java deleted file mode 100644 index 3c59923def..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ /dev/null @@ -1,313 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import com.google.protobuf.util.JsonFormat; -import jakarta.annotation.PreDestroy; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.queue.Queue; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; -import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; - -import java.nio.charset.StandardCharsets; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='monolith'") -public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { - - private final TopicService topicService; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbRabbitMqSettings rabbitMqSettings; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; - private final TbQueueVersionControlSettings vcSettings; - private final TbQueueEdgeSettings edgeSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - private final TbQueueAdmin vcAdmin; - private final TbQueueAdmin edgeAdmin; - - public RabbitMqMonolithQueueFactory(TopicService topicService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbRabbitMqSettings rabbitMqSettings, - TbRabbitMqQueueArguments queueArguments, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, - TbQueueVersionControlSettings vcSettings, - TbQueueEdgeSettings edgeSettings) { - this.topicService = topicService; - this.coreSettings = coreSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.rabbitMqSettings = rabbitMqSettings; - this.jsInvokeSettings = jsInvokeSettings; - this.vcSettings = vcSettings; - this.edgeSettings = edgeSettings; - - this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); - this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); - this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); - this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); - this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); - this.vcAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getVcArgs()); - this.edgeAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getEdgeArgs()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, - topicService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @Override - public TbQueueConsumer> createToVersionControlMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(vcAdmin, rabbitMqSettings, topicService.buildTopicName(vcSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) - ); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { - return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, topicService.buildTopicName(configuration.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, - topicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, - topicService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, topicService.buildTopicName(transportApiSettings.getRequestsTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, topicService.buildTopicName(transportApiSettings.getResponsesTopic())); - } - - @Override - @Bean - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbRabbitMqProducerTemplate<>(jsExecutorAdmin, rabbitMqSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbRabbitMqConsumerTemplate<>(jsExecutorAdmin, rabbitMqSettings, - jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> { - RemoteJsResponse.Builder builder = RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); - } - - @Override - public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToOtaPackageStateServiceMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToOtaPackageStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); - } - - @Override - public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); - } - - @Override - public TbQueueProducer> createVersionControlMsgProducer() { - return new TbRabbitMqProducerTemplate<>(vcAdmin, rabbitMqSettings, topicService.buildTopicName(vcSettings.getTopic())); - } - - @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); - } - - @Override - public TbQueueConsumer> createHousekeeperMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); - } - - @Override - public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createEdgeMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(edgeAdmin, rabbitMqSettings, topicService.buildTopicName(edgeSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createEdgeMsgProducer() { - return new TbRabbitMqProducerTemplate<>(edgeAdmin, rabbitMqSettings, topicService.buildTopicName(edgeSettings.getTopic())); - } - - @Override - public TbQueueConsumer> createToEdgeNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, - topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createEdgeNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, - topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @Override - public TbQueueProducer> createEdgeEventMsgProducer() { - return null; - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - if (vcAdmin != null) { - vcAdmin.destroy(); - } - if (edgeAdmin != null) { - edgeAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java deleted file mode 100644 index 16f9838384..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ /dev/null @@ -1,278 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import com.google.protobuf.util.JsonFormat; -import jakarta.annotation.PreDestroy; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; - -import java.nio.charset.StandardCharsets; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-core'") -public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { - - private final TbRabbitMqSettings rabbitMqSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueCoreSettings coreSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TopicService topicService; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbQueueEdgeSettings edgeSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - private final TbQueueAdmin edgeAdmin; - - public RabbitMqTbCoreQueueFactory(TbRabbitMqSettings rabbitMqSettings, - TbQueueCoreSettings coreSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TopicService topicService, - TbServiceInfoProvider serviceInfoProvider, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbQueueEdgeSettings edgeSettings, - TbRabbitMqQueueArguments queueArguments) { - this.rabbitMqSettings = rabbitMqSettings; - this.coreSettings = coreSettings; - this.transportApiSettings = transportApiSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.topicService = topicService; - this.serviceInfoProvider = serviceInfoProvider; - this.jsInvokeSettings = jsInvokeSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.edgeSettings = edgeSettings; - - this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); - this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); - this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); - this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); - this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); - this.edgeAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getEdgeArgs()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, - topicService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, - topicService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createEdgeMsgProducer() { - return new TbRabbitMqProducerTemplate<>(edgeAdmin, rabbitMqSettings, topicService.buildTopicName(edgeSettings.getTopic())); - } - - @Override - public TbQueueConsumer> createEdgeMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(edgeAdmin, rabbitMqSettings, topicService.buildTopicName(edgeSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToEdgeNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, - topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createEdgeNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, - topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, topicService.buildTopicName(transportApiSettings.getRequestsTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, topicService.buildTopicName(transportApiSettings.getResponsesTopic())); - } - - @Override - @Bean - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbRabbitMqProducerTemplate<>(jsExecutorAdmin, rabbitMqSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbRabbitMqConsumerTemplate<>(jsExecutorAdmin, rabbitMqSettings, - jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); - } - - @Override - public TbQueueProducer> createVersionControlMsgProducer() { - //TODO: version-control - return null; - } - - @Override - public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToOtaPackageStateServiceMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToOtaPackageStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); - } - - @Override - public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); - } - - @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); - } - - @Override - public TbQueueConsumer> createHousekeeperMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); - } - - @Override - public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - if (edgeAdmin != null) { - edgeAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java deleted file mode 100644 index a7af729e1f..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java +++ /dev/null @@ -1,202 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import com.google.protobuf.util.JsonFormat; -import jakarta.annotation.PreDestroy; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.queue.Queue; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; - -import java.nio.charset.StandardCharsets; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-rule-engine'") -public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { - - private final TopicService topicService; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbRabbitMqSettings rabbitMqSettings; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbQueueEdgeSettings edgeSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin notificationAdmin; - private final TbQueueAdmin edgeAdmin; - - public RabbitMqTbRuleEngineQueueFactory(TopicService topicService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbRabbitMqSettings rabbitMqSettings, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbQueueEdgeSettings edgeSettings, - TbRabbitMqQueueArguments queueArguments) { - this.topicService = topicService; - this.coreSettings = coreSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.rabbitMqSettings = rabbitMqSettings; - this.jsInvokeSettings = jsInvokeSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.edgeSettings = edgeSettings; - - this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); - this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); - this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); - this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); - this.edgeAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getEdgeArgs()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createEdgeMsgProducer() { - return new TbRabbitMqProducerTemplate<>(edgeAdmin, rabbitMqSettings, topicService.buildTopicName(edgeSettings.getTopic())); - } - - @Override - public TbQueueProducer> createEdgeNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { - return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, topicService.buildTopicName(configuration.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, - topicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - @Bean - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbRabbitMqProducerTemplate<>(jsExecutorAdmin, rabbitMqSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbRabbitMqConsumerTemplate<>(jsExecutorAdmin, rabbitMqSettings, - jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); - } - - @Override - public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); - } - - @Override - public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); - } - - @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbVersionControlQueueFactory.java deleted file mode 100644 index 604955be2c..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbVersionControlQueueFactory.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import jakarta.annotation.PreDestroy; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-vc-executor'") -public class RabbitMqTbVersionControlQueueFactory implements TbVersionControlQueueFactory { - - private final TbRabbitMqSettings rabbitMqSettings; - private final TbQueueCoreSettings coreSettings; - private final TbQueueVersionControlSettings vcSettings; - private final TopicService topicService; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin notificationAdmin; - private final TbQueueAdmin vcAdmin; - - public RabbitMqTbVersionControlQueueFactory(TbRabbitMqSettings rabbitMqSettings, - TbQueueCoreSettings coreSettings, - TbQueueVersionControlSettings vcSettings, - TbRabbitMqQueueArguments queueArguments, - TopicService topicService - ) { - this.rabbitMqSettings = rabbitMqSettings; - this.coreSettings = coreSettings; - this.vcSettings = vcSettings; - this.topicService = topicService; - - this.coreAdmin = new TbRabbitMqAdmin(this.rabbitMqSettings, queueArguments.getCoreArgs()); - this.notificationAdmin = new TbRabbitMqAdmin(this.rabbitMqSettings, queueArguments.getNotificationsArgs()); - this.vcAdmin = new TbRabbitMqAdmin(this.rabbitMqSettings, queueArguments.getVcArgs()); - } - - @Override - public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueConsumer> createToVersionControlMsgConsumer() { - return new TbRabbitMqConsumerTemplate<>(vcAdmin, rabbitMqSettings, topicService.buildTopicName(vcSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) - ); - } - - @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - if (vcAdmin != null) { - vcAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java deleted file mode 100644 index ea922557bd..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java +++ /dev/null @@ -1,154 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import jakarta.annotation.PreDestroy; -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; -import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && (('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true') || '${service.type:null}'=='tb-transport')") -@Slf4j -public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory { - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbRabbitMqSettings rabbitMqSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueCoreSettings coreSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TopicService topicService; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - - public RabbitMqTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbRabbitMqSettings rabbitMqSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbRabbitMqQueueArguments queueArguments, - TopicService topicService) { - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.rabbitMqSettings = rabbitMqSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.coreSettings = coreSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.topicService = topicService; - - this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); - this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); - this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); - this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbQueueProducer> producerTemplate = - new TbRabbitMqProducerTemplate<>(transportApiAdmin, rabbitMqSettings, topicService.buildTopicName(transportApiSettings.getRequestsTopic())); - - TbQueueConsumer> consumerTemplate = - new TbRabbitMqConsumerTemplate<>(transportApiAdmin, rabbitMqSettings, - topicService.buildTopicName(transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(transportApiAdmin); - templateBuilder.requestTemplate(producerTemplate); - templateBuilder.responseTemplate(consumerTemplate); - templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); - templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); - templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); - return templateBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbRabbitMqProducerTemplate<>(ruleEngineAdmin, rabbitMqSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueConsumer> createTransportNotificationsConsumer() { - return new TbRabbitMqConsumerTemplate<>(notificationAdmin, rabbitMqSettings, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); - } - - @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java deleted file mode 100644 index 932bbd21e5..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ /dev/null @@ -1,312 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import com.google.protobuf.util.JsonFormat; -import jakarta.annotation.PreDestroy; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.queue.Queue; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; -import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; - -import java.nio.charset.StandardCharsets; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='monolith'") -public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { - - private final TopicService topicService; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbServiceBusSettings serviceBusSettings; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; - private final TbQueueVersionControlSettings vcSettings; - private final TbQueueEdgeSettings edgeSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - private final TbQueueAdmin vcAdmin; - private final TbQueueAdmin edgeAdmin; - - public ServiceBusMonolithQueueFactory(TopicService topicService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbServiceBusSettings serviceBusSettings, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, - TbQueueVersionControlSettings vcSettings, - TbQueueEdgeSettings edgeSettings, - TbServiceBusQueueConfigs serviceBusQueueConfigs) { - this.topicService = topicService; - this.coreSettings = coreSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.serviceBusSettings = serviceBusSettings; - this.jsInvokeSettings = jsInvokeSettings; - this.vcSettings = vcSettings; - this.edgeSettings = edgeSettings; - - this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); - this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); - this.jsExecutorAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getJsExecutorConfigs()); - this.transportApiAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getTransportApiConfigs()); - this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs()); - this.vcAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getVcConfigs()); - this.edgeAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getEdgeConfigs()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, - topicService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @Override - public TbQueueConsumer> createToVersionControlMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(vcAdmin, serviceBusSettings, topicService.buildTopicName(vcSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) - ); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { - return new TbServiceBusConsumerTemplate<>(ruleEngineAdmin, serviceBusSettings, configuration.getTopic(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, - topicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, - topicService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbServiceBusConsumerTemplate<>(transportApiAdmin, serviceBusSettings, topicService.buildTopicName(transportApiSettings.getRequestsTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, topicService.buildTopicName(transportApiSettings.getResponsesTopic())); - } - - @Override - @Bean - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbServiceBusProducerTemplate<>(jsExecutorAdmin, serviceBusSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbServiceBusConsumerTemplate<>(jsExecutorAdmin, serviceBusSettings, - jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); - } - - @Override - public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToOtaPackageStateServiceMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToOtaPackageStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { - return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); - } - - @Override - public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); - } - - @Override - public TbQueueProducer> createVersionControlMsgProducer() { - return new TbServiceBusProducerTemplate<>(vcAdmin, serviceBusSettings, topicService.buildTopicName(vcSettings.getTopic())); - } - - @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); - } - - @Override - public TbQueueConsumer> createHousekeeperMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { - return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); - } - - @Override - public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createEdgeMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(edgeAdmin, serviceBusSettings, topicService.buildTopicName(edgeSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createEdgeMsgProducer() { - return new TbServiceBusProducerTemplate<>(edgeAdmin, serviceBusSettings, topicService.buildTopicName(edgeSettings.getTopic())); - } - - @Override - public TbQueueConsumer> createToEdgeNotificationsMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, - topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createEdgeNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, - topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @Override - public TbQueueProducer> createEdgeEventMsgProducer() { - return null; - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - if (vcAdmin != null) { - vcAdmin.destroy(); - } - if (edgeAdmin != null) { - edgeAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java deleted file mode 100644 index 71a7efe50b..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ /dev/null @@ -1,278 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import com.google.protobuf.util.JsonFormat; -import jakarta.annotation.PreDestroy; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; - -import java.nio.charset.StandardCharsets; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-core'") -public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { - - private final TbServiceBusSettings serviceBusSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbQueueCoreSettings coreSettings; - private final TbQueueTransportApiSettings transportApiSettings; - private final TopicService topicService; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbQueueEdgeSettings edgeSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - private final TbQueueAdmin edgeAdmin; - - public ServiceBusTbCoreQueueFactory(TbServiceBusSettings serviceBusSettings, - TbQueueCoreSettings coreSettings, - TbQueueTransportApiSettings transportApiSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TopicService topicService, - TbServiceInfoProvider serviceInfoProvider, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbQueueEdgeSettings edgeSettings, - TbServiceBusQueueConfigs serviceBusQueueConfigs) { - this.serviceBusSettings = serviceBusSettings; - this.coreSettings = coreSettings; - this.transportApiSettings = transportApiSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.topicService = topicService; - this.serviceInfoProvider = serviceInfoProvider; - this.jsInvokeSettings = jsInvokeSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.edgeSettings = edgeSettings; - - this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); - this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); - this.jsExecutorAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getJsExecutorConfigs()); - this.transportApiAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getTransportApiConfigs()); - this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs()); - this.edgeAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getEdgeConfigs()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, - topicService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @Override - public TbQueueConsumer> createToCoreMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToCoreNotificationsMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, - topicService.getNotificationsTopic(ServiceType.TB_CORE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCoreNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createTransportApiRequestConsumer() { - return new TbServiceBusConsumerTemplate<>(transportApiAdmin, serviceBusSettings, topicService.buildTopicName(transportApiSettings.getRequestsTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiRequestMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createTransportApiResponseProducer() { - return new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, topicService.buildTopicName(transportApiSettings.getResponsesTopic())); - } - - @Override - @Bean - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbServiceBusProducerTemplate<>(jsExecutorAdmin, serviceBusSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbServiceBusConsumerTemplate<>(jsExecutorAdmin, serviceBusSettings, - jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); - } - - @Override - public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToUsageStatsServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToOtaPackageStateServiceMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToOtaPackageStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { - return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); - } - - @Override - public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); - } - - @Override - public TbQueueProducer> createVersionControlMsgProducer() { - //TODO: version-control - return null; - } - - @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); - } - - @Override - public TbQueueConsumer> createHousekeeperMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { - return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); - } - - @Override - public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createEdgeMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(edgeAdmin, serviceBusSettings, topicService.buildTopicName(edgeSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createEdgeMsgProducer() { - return new TbServiceBusProducerTemplate<>(edgeAdmin, serviceBusSettings, topicService.buildTopicName(edgeSettings.getTopic())); - } - - @Override - public TbQueueConsumer> createToEdgeNotificationsMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, - topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createEdgeNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, - topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - if (edgeAdmin != null) { - edgeAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java deleted file mode 100644 index 54661f695c..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java +++ /dev/null @@ -1,202 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import com.google.protobuf.util.JsonFormat; -import jakarta.annotation.PreDestroy; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.queue.Queue; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; -import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; - -import java.nio.charset.StandardCharsets; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-rule-engine'") -public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { - - private final TopicService topicService; - private final TbQueueCoreSettings coreSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TbServiceBusSettings serviceBusSettings; - private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbQueueEdgeSettings edgeSettings; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin ruleEngineAdmin; - private final TbQueueAdmin jsExecutorAdmin; - private final TbQueueAdmin notificationAdmin; - private final TbQueueAdmin edgeAdmin; - - public ServiceBusTbRuleEngineQueueFactory(TopicService topicService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbServiceBusSettings serviceBusSettings, - TbQueueRemoteJsInvokeSettings jsInvokeSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbQueueEdgeSettings edgeSettings, - TbServiceBusQueueConfigs serviceBusQueueConfigs) { - this.topicService = topicService; - this.coreSettings = coreSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.serviceBusSettings = serviceBusSettings; - this.jsInvokeSettings = jsInvokeSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.edgeSettings = edgeSettings; - - this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); - this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); - this.jsExecutorAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getJsExecutorConfigs()); - this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs()); - this.edgeAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getEdgeConfigs()); - } - - @Override - public TbQueueProducer> createTransportNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createRuleEngineNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createEdgeMsgProducer() { - return new TbServiceBusProducerTemplate<>(edgeAdmin, serviceBusSettings, topicService.buildTopicName(edgeSettings.getTopic())); - } - - @Override - public TbQueueProducer> createEdgeNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, topicService.getEdgeNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); - } - - @Override - public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { - return new TbServiceBusConsumerTemplate<>(ruleEngineAdmin, serviceBusSettings, topicService.buildTopicName(configuration.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, - topicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceInfoProvider.getServiceId()).getFullTopicName(), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - @Bean - public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { - TbQueueProducer> producer = new TbServiceBusProducerTemplate<>(jsExecutorAdmin, serviceBusSettings, jsInvokeSettings.getRequestTopic()); - TbQueueConsumer> consumer = new TbServiceBusConsumerTemplate<>(jsExecutorAdmin, serviceBusSettings, - jsInvokeSettings.getResponseTopic() + "." + serviceInfoProvider.getServiceId(), - msg -> { - JsInvokeProtos.RemoteJsResponse.Builder builder = JsInvokeProtos.RemoteJsResponse.newBuilder(); - JsonFormat.parser().ignoringUnknownFields().merge(new String(msg.getData(), StandardCharsets.UTF_8), builder); - return new TbProtoQueueMsg<>(msg.getKey(), builder.build(), msg.getHeaders()); - }); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> builder = DefaultTbQueueRequestTemplate.builder(); - builder.queueAdmin(jsExecutorAdmin); - builder.requestTemplate(producer); - builder.responseTemplate(consumer); - builder.maxPendingRequests(jsInvokeSettings.getMaxPendingRequests()); - builder.maxRequestTimeout(jsInvokeSettings.getMaxRequestsTimeout()); - builder.pollInterval(jsInvokeSettings.getResponsePollInterval()); - return builder.build(); - } - - @Override - public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); - } - - @Override - public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { - return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); - } - - @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - if (jsExecutorAdmin != null) { - jsExecutorAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbVersionControlQueueFactory.java deleted file mode 100644 index 0c363488ce..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbVersionControlQueueFactory.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import jakarta.annotation.PreDestroy; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-vc-executor'") -public class ServiceBusTbVersionControlQueueFactory implements TbVersionControlQueueFactory { - - private final TbServiceBusSettings serviceBusSettings; - private final TbQueueCoreSettings coreSettings; - private final TbQueueVersionControlSettings vcSettings; - private final TopicService topicService; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin notificationAdmin; - private final TbQueueAdmin vcAdmin; - - public ServiceBusTbVersionControlQueueFactory(TbServiceBusSettings serviceBusSettings, - TbQueueCoreSettings coreSettings, - TbQueueVersionControlSettings vcSettings, - TbServiceBusQueueConfigs serviceBusQueueConfigs, - TopicService topicService - ) { - this.serviceBusSettings = serviceBusSettings; - this.coreSettings = coreSettings; - this.vcSettings = vcSettings; - this.topicService = topicService; - - this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); - this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs()); - this.vcAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getVcConfigs()); - } - - @Override - public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueConsumer> createToVersionControlMsgConsumer() { - return new TbServiceBusConsumerTemplate<>(vcAdmin, serviceBusSettings, topicService.buildTopicName(vcSettings.getTopic()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) - ); - } - - @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - if (vcAdmin != null) { - vcAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java deleted file mode 100644 index 7f3e246eec..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java +++ /dev/null @@ -1,155 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.provider; - -import jakarta.annotation.PreDestroy; -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.TbQueueRequestTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; -import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; -import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; -import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.discovery.TopicService; -import org.thingsboard.server.queue.settings.TbQueueCoreSettings; -import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; -import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && (('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true') || '${service.type:null}'=='tb-transport')") -@Slf4j -public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory { - private final TbQueueTransportApiSettings transportApiSettings; - private final TbQueueTransportNotificationSettings transportNotificationSettings; - private final TbServiceBusSettings serviceBusSettings; - private final TbServiceInfoProvider serviceInfoProvider; - private final TbQueueCoreSettings coreSettings; - private final TbQueueRuleEngineSettings ruleEngineSettings; - private final TopicService topicService; - - private final TbQueueAdmin coreAdmin; - private final TbQueueAdmin transportApiAdmin; - private final TbQueueAdmin notificationAdmin; - private final TbQueueAdmin ruleEngineAdmin; - - public ServiceBusTransportQueueFactory(TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbServiceBusSettings serviceBusSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueCoreSettings coreSettings, - TbServiceBusQueueConfigs serviceBusQueueConfigs, - TopicService topicService) { - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.serviceBusSettings = serviceBusSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.coreSettings = coreSettings; - this.ruleEngineSettings = ruleEngineSettings; - this.topicService = topicService; - - this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); - this.transportApiAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getTransportApiConfigs()); - this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs()); - this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); - } - - @Override - public TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate() { - TbQueueProducer> producerTemplate = - new TbServiceBusProducerTemplate<>(transportApiAdmin, serviceBusSettings, topicService.buildTopicName(transportApiSettings.getRequestsTopic())); - - TbQueueConsumer> consumerTemplate = - new TbServiceBusConsumerTemplate<>(transportApiAdmin, serviceBusSettings, - topicService.buildTopicName(transportApiSettings.getResponsesTopic() + "." + serviceInfoProvider.getServiceId()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportApiResponseMsg.parseFrom(msg.getData()), msg.getHeaders())); - - DefaultTbQueueRequestTemplate.DefaultTbQueueRequestTemplateBuilder - , TbProtoQueueMsg> templateBuilder = DefaultTbQueueRequestTemplate.builder(); - templateBuilder.queueAdmin(transportApiAdmin); - templateBuilder.requestTemplate(producerTemplate); - templateBuilder.responseTemplate(consumerTemplate); - templateBuilder.maxPendingRequests(transportApiSettings.getMaxPendingRequests()); - templateBuilder.maxRequestTimeout(transportApiSettings.getMaxRequestsTimeout()); - templateBuilder.pollInterval(transportApiSettings.getResponsePollInterval()); - return templateBuilder.build(); - } - - @Override - public TbQueueProducer> createRuleEngineMsgProducer() { - return new TbServiceBusProducerTemplate<>(ruleEngineAdmin, serviceBusSettings, topicService.buildTopicName(ruleEngineSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreMsgProducer() { - return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueProducer> createTbCoreNotificationsMsgProducer() { - return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getTopic())); - } - - @Override - public TbQueueConsumer> createTransportNotificationsConsumer() { - return new TbServiceBusConsumerTemplate<>(notificationAdmin, serviceBusSettings, - topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic() + "." + serviceInfoProvider.getServiceId()), - msg -> new TbProtoQueueMsg<>(msg.getKey(), ToTransportMsg.parseFrom(msg.getData()), msg.getHeaders())); - } - - @Override - public TbQueueProducer> createToUsageStatsServiceMsgProducer() { - return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); - } - - @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); - } - - @PreDestroy - private void destroy() { - if (coreAdmin != null) { - coreAdmin.destroy(); - } - if (transportApiAdmin != null) { - transportApiAdmin.destroy(); - } - if (notificationAdmin != null) { - notificationAdmin.destroy(); - } - if (ruleEngineAdmin != null) { - ruleEngineAdmin.destroy(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java deleted file mode 100644 index bbae199c4a..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java +++ /dev/null @@ -1,232 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.pubsub; - -import com.google.api.gax.rpc.AlreadyExistsException; -import com.google.cloud.pubsub.v1.SubscriptionAdminClient; -import com.google.cloud.pubsub.v1.SubscriptionAdminSettings; -import com.google.cloud.pubsub.v1.TopicAdminClient; -import com.google.cloud.pubsub.v1.TopicAdminSettings; -import com.google.protobuf.Duration; -import com.google.pubsub.v1.ListSubscriptionsRequest; -import com.google.pubsub.v1.ListTopicsRequest; -import com.google.pubsub.v1.ProjectName; -import com.google.pubsub.v1.ProjectSubscriptionName; -import com.google.pubsub.v1.Subscription; -import com.google.pubsub.v1.Topic; -import com.google.pubsub.v1.TopicName; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.queue.TbQueueAdmin; - -import java.io.IOException; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - -@Slf4j -@Deprecated(forRemoval = true, since = "3.9") // for removal in 4.0 -public class TbPubSubAdmin implements TbQueueAdmin { - private static final String ACK_DEADLINE = "ackDeadlineInSec"; - private static final String MESSAGE_RETENTION = "messageRetentionInSec"; - - private final TopicAdminClient topicAdminClient; - private final SubscriptionAdminClient subscriptionAdminClient; - - private final TbPubSubSettings pubSubSettings; - private final Set topicSet = ConcurrentHashMap.newKeySet(); - private final Set subscriptionSet = ConcurrentHashMap.newKeySet(); - private final Map subscriptionProperties; - - public TbPubSubAdmin(TbPubSubSettings pubSubSettings, Map subscriptionSettings) { - this.pubSubSettings = pubSubSettings; - this.subscriptionProperties = subscriptionSettings; - - TopicAdminSettings topicAdminSettings; - try { - topicAdminSettings = TopicAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); - } catch (IOException e) { - log.error("Failed to create TopicAdminSettings"); - throw new RuntimeException("Failed to create TopicAdminSettings."); - } - - SubscriptionAdminSettings subscriptionAdminSettings; - try { - subscriptionAdminSettings = SubscriptionAdminSettings.newBuilder().setCredentialsProvider(pubSubSettings.getCredentialsProvider()).build(); - } catch (IOException e) { - log.error("Failed to create SubscriptionAdminSettings"); - throw new RuntimeException("Failed to create SubscriptionAdminSettings."); - } - - try { - topicAdminClient = TopicAdminClient.create(topicAdminSettings); - - ListTopicsRequest listTopicsRequest = - ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); - TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); - for (Topic topic : response.iterateAll()) { - topicSet.add(topic.getName()); - } - } catch (IOException e) { - log.error("Failed to get topics.", e); - throw new RuntimeException("Failed to get topics.", e); - } - - try { - subscriptionAdminClient = SubscriptionAdminClient.create(subscriptionAdminSettings); - - ListSubscriptionsRequest listSubscriptionsRequest = - ListSubscriptionsRequest.newBuilder() - .setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()) - .build(); - SubscriptionAdminClient.ListSubscriptionsPagedResponse response = - subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest); - - for (Subscription subscription : response.iterateAll()) { - subscriptionSet.add(subscription.getName()); - } - } catch (IOException e) { - log.error("Failed to get subscriptions.", e); - throw new RuntimeException("Failed to get subscriptions.", e); - } - } - - @Override - public void createTopicIfNotExists(String partition, String properties) { - TopicName topicName = TopicName.newBuilder() - .setTopic(partition) - .setProject(pubSubSettings.getProjectId()) - .build(); - - if (topicSet.contains(topicName.toString())) { - createSubscriptionIfNotExists(partition, topicName); - return; - } - - ListTopicsRequest listTopicsRequest = - ListTopicsRequest.newBuilder().setProject(ProjectName.format(pubSubSettings.getProjectId())).build(); - TopicAdminClient.ListTopicsPagedResponse response = topicAdminClient.listTopics(listTopicsRequest); - for (Topic topic : response.iterateAll()) { - if (topic.getName().contains(topicName.toString())) { - topicSet.add(topic.getName()); - createSubscriptionIfNotExists(partition, topicName); - return; - } - } - - try { - topicAdminClient.createTopic(topicName); - log.info("Created new topic: [{}]", topicName.toString()); - } catch (AlreadyExistsException e) { - log.info("[{}] Topic already exist.", topicName.toString()); - } finally { - topicSet.add(topicName.toString()); - } - createSubscriptionIfNotExists(partition, topicName); - } - - @Override - public void deleteTopic(String topic) { - TopicName topicName = TopicName.newBuilder() - .setTopic(topic) - .setProject(pubSubSettings.getProjectId()) - .build(); - - ProjectSubscriptionName subscriptionName = - ProjectSubscriptionName.of(pubSubSettings.getProjectId(), topic); - - if (topicSet.contains(topicName.toString())) { - topicAdminClient.deleteTopic(topicName); - } else { - if (topicAdminClient.getTopic(topicName) != null) { - topicAdminClient.deleteTopic(topicName); - } else { - log.warn("PubSub topic [{}] does not exist.", topic); - } - } - - if (subscriptionSet.contains(subscriptionName.toString())) { - subscriptionAdminClient.deleteSubscription(subscriptionName); - } else { - if (subscriptionAdminClient.getSubscription(subscriptionName) != null) { - subscriptionAdminClient.deleteSubscription(subscriptionName); - } else { - log.warn("PubSub subscription [{}] does not exist.", topic); - } - } - } - - private void createSubscriptionIfNotExists(String partition, TopicName topicName) { - ProjectSubscriptionName subscriptionName = - ProjectSubscriptionName.of(pubSubSettings.getProjectId(), partition); - - if (subscriptionSet.contains(subscriptionName.toString())) { - return; - } - - ListSubscriptionsRequest listSubscriptionsRequest = - ListSubscriptionsRequest.newBuilder().setProject(ProjectName.of(pubSubSettings.getProjectId()).toString()).build(); - SubscriptionAdminClient.ListSubscriptionsPagedResponse response = subscriptionAdminClient.listSubscriptions(listSubscriptionsRequest); - for (Subscription subscription : response.iterateAll()) { - if (subscription.getName().equals(subscriptionName.toString())) { - subscriptionSet.add(subscription.getName()); - return; - } - } - - Subscription.Builder subscriptionBuilder = Subscription - .newBuilder() - .setName(subscriptionName.toString()) - .setTopic(topicName.toString()); - - setAckDeadline(subscriptionBuilder); - setMessageRetention(subscriptionBuilder); - - try { - subscriptionAdminClient.createSubscription(subscriptionBuilder.build()); - log.info("Created new subscription: [{}]", subscriptionName.toString()); - } catch (AlreadyExistsException e) { - log.info("[{}] Subscription already exist.", subscriptionName.toString()); - } finally { - subscriptionSet.add(subscriptionName.toString()); - } - } - - private void setAckDeadline(Subscription.Builder builder) { - if (subscriptionProperties.containsKey(ACK_DEADLINE)) { - builder.setAckDeadlineSeconds(Integer.parseInt(subscriptionProperties.get(ACK_DEADLINE))); - } - } - - private void setMessageRetention(Subscription.Builder builder) { - if (subscriptionProperties.containsKey(MESSAGE_RETENTION)) { - Duration duration = Duration - .newBuilder() - .setSeconds(Long.parseLong(subscriptionProperties.get(MESSAGE_RETENTION))) - .build(); - builder.setMessageRetentionDuration(duration); - } - } - - @Override - public void destroy() { - if (topicAdminClient != null) { - topicAdminClient.close(); - } - if (subscriptionAdminClient != null) { - subscriptionAdminClient.close(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java deleted file mode 100644 index 8a2ba90096..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java +++ /dev/null @@ -1,175 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.pubsub; - -import com.google.api.core.ApiFuture; -import com.google.api.core.ApiFutures; -import com.google.cloud.pubsub.v1.stub.GrpcSubscriberStub; -import com.google.cloud.pubsub.v1.stub.SubscriberStub; -import com.google.cloud.pubsub.v1.stub.SubscriberStubSettings; -import com.google.gson.Gson; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.pubsub.v1.AcknowledgeRequest; -import com.google.pubsub.v1.ProjectSubscriptionName; -import com.google.pubsub.v1.PubsubMessage; -import com.google.pubsub.v1.PullRequest; -import com.google.pubsub.v1.PullResponse; -import com.google.pubsub.v1.ReceivedMessage; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.CollectionUtils; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueMsgDecoder; -import org.thingsboard.server.queue.common.AbstractParallelTbQueueConsumerTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; - -@Slf4j -public class TbPubSubConsumerTemplate extends AbstractParallelTbQueueConsumerTemplate { - - private final Gson gson = new Gson(); - private final TbQueueAdmin admin; - private final String topic; - private final TbQueueMsgDecoder decoder; - private final TbPubSubSettings pubSubSettings; - - private volatile Set subscriptionNames; - private final List acknowledgeRequests = new CopyOnWriteArrayList<>(); - - private final SubscriberStub subscriber; - private volatile int messagesPerTopic; - - public TbPubSubConsumerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String topic, TbQueueMsgDecoder decoder) { - super(topic); - this.admin = admin; - this.pubSubSettings = pubSubSettings; - this.topic = topic; - this.decoder = decoder; - try { - SubscriberStubSettings subscriberStubSettings = - SubscriberStubSettings.newBuilder() - .setCredentialsProvider(pubSubSettings.getCredentialsProvider()) - .setTransportChannelProvider( - SubscriberStubSettings.defaultGrpcTransportProviderBuilder() - .setMaxInboundMessageSize(pubSubSettings.getMaxMsgSize()) - .build()) - .setExecutorProvider(pubSubSettings.getExecutorProvider()) - .build(); - this.subscriber = GrpcSubscriberStub.create(subscriberStubSettings); - } catch (IOException e) { - log.error("Failed to create subscriber.", e); - throw new RuntimeException("Failed to create subscriber.", e); - } - } - - @Override - protected List doPoll(long durationInMillis) { - try { - List messages = receiveMessages(); - if (!messages.isEmpty()) { - return messages.stream().map(ReceivedMessage::getMessage).collect(Collectors.toList()); - } - } catch (ExecutionException | InterruptedException e) { - if (stopped) { - log.info("[{}] Pub/Sub consumer is stopped.", topic); - } else { - log.error("Failed to receive messages", e); - } - } - return Collections.emptyList(); - } - - @Override - protected void doSubscribe(List topicNames) { - subscriptionNames = new LinkedHashSet<>(topicNames); - subscriptionNames.forEach(admin::createTopicIfNotExists); - initNewExecutor(subscriptionNames.size() + 1); - messagesPerTopic = pubSubSettings.getMaxMessages() / Math.max(subscriptionNames.size(), 1); - } - - @Override - protected void doCommit() { - acknowledgeRequests.forEach(subscriber.acknowledgeCallable()::futureCall); - acknowledgeRequests.clear(); - } - - @Override - protected void doUnsubscribe() { - if (subscriber != null) { - subscriber.close(); - } - shutdownExecutor(); - } - - private List receiveMessages() throws ExecutionException, InterruptedException { - List>> result = subscriptionNames.stream().map(subscriptionId -> { - String subscriptionName = ProjectSubscriptionName.format(pubSubSettings.getProjectId(), subscriptionId); - PullRequest pullRequest = - PullRequest.newBuilder() - .setMaxMessages(messagesPerTopic) -// .setReturnImmediately(false) // return immediately if messages are not available - .setSubscription(subscriptionName) - .build(); - - ApiFuture pullResponseApiFuture = subscriber.pullCallable().futureCall(pullRequest); - - return ApiFutures.transform(pullResponseApiFuture, pullResponse -> { - if (pullResponse != null && !pullResponse.getReceivedMessagesList().isEmpty()) { - List ackIds = new ArrayList<>(); - for (ReceivedMessage message : pullResponse.getReceivedMessagesList()) { - ackIds.add(message.getAckId()); - } - AcknowledgeRequest acknowledgeRequest = - AcknowledgeRequest.newBuilder() - .setSubscription(subscriptionName) - .addAllAckIds(ackIds) - .build(); - - acknowledgeRequests.add(acknowledgeRequest); - return pullResponse.getReceivedMessagesList(); - } - return null; - }, consumerExecutor); - - }).collect(Collectors.toList()); - - ApiFuture> transform = ApiFutures.transform(ApiFutures.allAsList(result), listMessages -> { - if (!CollectionUtils.isEmpty(listMessages)) { - return listMessages.stream().filter(Objects::nonNull).flatMap(List::stream).collect(Collectors.toList()); - } - return Collections.emptyList(); - }, consumerExecutor); - - return transform.get(); - } - - @Override - public T decode(PubsubMessage message) throws InvalidProtocolBufferException { - DefaultTbQueueMsg msg = gson.fromJson(message.getData().toStringUtf8(), DefaultTbQueueMsg.class); - return decoder.decode(msg); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java deleted file mode 100644 index bb1d4e8976..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java +++ /dev/null @@ -1,137 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.pubsub; - -import com.google.api.core.ApiFuture; -import com.google.api.core.ApiFutureCallback; -import com.google.api.core.ApiFutures; -import com.google.cloud.pubsub.v1.Publisher; -import com.google.gson.Gson; -import com.google.protobuf.ByteString; -import com.google.pubsub.v1.ProjectTopicName; -import com.google.pubsub.v1.PubsubMessage; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.io.IOException; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -@Slf4j -public class TbPubSubProducerTemplate implements TbQueueProducer { - - private final Gson gson = new Gson(); - - private final String defaultTopic; - private final TbQueueAdmin admin; - private final TbPubSubSettings pubSubSettings; - - private final Map publisherMap = new ConcurrentHashMap<>(); - - private final ExecutorService pubExecutor = Executors.newCachedThreadPool(); - - public TbPubSubProducerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String defaultTopic) { - this.defaultTopic = defaultTopic; - this.admin = admin; - this.pubSubSettings = pubSubSettings; - } - - @Override - public void init() { - - } - - @Override - public String getDefaultTopic() { - return defaultTopic; - } - - @Override - public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { - PubsubMessage.Builder pubsubMessageBuilder = PubsubMessage.newBuilder(); - pubsubMessageBuilder.setData(getMsg(msg)); - - Publisher publisher = getOrCreatePublisher(tpi.getFullTopicName()); - ApiFuture future = publisher.publish(pubsubMessageBuilder.build()); - - ApiFutures.addCallback(future, new ApiFutureCallback() { - public void onSuccess(String messageId) { - if (callback != null) { - callback.onSuccess(null); - } - } - - public void onFailure(Throwable t) { - if (callback != null) { - callback.onFailure(t); - } - } - }, pubExecutor); - } - - @Override - public void stop() { - publisherMap.forEach((k, v) -> { - if (v != null) { - try { - v.shutdown(); - v.awaitTermination(1, TimeUnit.SECONDS); - } catch (Exception e) { - log.error("Failed to shutdown PubSub client during destroy()", e); - } - } - }); - - if (pubExecutor != null) { - pubExecutor.shutdownNow(); - } - } - - private ByteString getMsg(T msg) { - String json = gson.toJson(new DefaultTbQueueMsg(msg)); - return ByteString.copyFrom(json.getBytes()); - } - - private Publisher getOrCreatePublisher(String topic) { - if (publisherMap.containsKey(topic)) { - return publisherMap.get(topic); - } else { - try { - admin.createTopicIfNotExists(topic); - ProjectTopicName topicName = ProjectTopicName.of(pubSubSettings.getProjectId(), topic); - Publisher publisher = Publisher.newBuilder(topicName) - .setCredentialsProvider(pubSubSettings.getCredentialsProvider()) - .setExecutorProvider(pubSubSettings.getExecutorProvider()) - .build(); - publisherMap.put(topic, publisher); - return publisher; - } catch (IOException e) { - log.error("Failed to create Publisher for the topic [{}].", topic, e); - throw new RuntimeException("Failed to create Publisher for the topic.", e); - } - } - - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java deleted file mode 100644 index 4794739925..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSettings.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.pubsub; - -import com.google.api.gax.core.CredentialsProvider; -import com.google.api.gax.core.FixedCredentialsProvider; -import com.google.api.gax.core.FixedExecutorProvider; -import com.google.auth.oauth2.ServiceAccountCredentials; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.common.util.ThingsBoardExecutors; - -import java.io.ByteArrayInputStream; -import java.io.IOException; - -@Slf4j -@ConditionalOnExpression("'${queue.type:null}'=='pubsub'") -@Component -@Data -public class TbPubSubSettings { - - @Value("${queue.pubsub.project_id}") - private String projectId; - - @Value("${queue.pubsub.service_account}") - private String serviceAccount; - - @Value("${queue.pubsub.max_msg_size}") - private int maxMsgSize; - - @Value("${queue.pubsub.max_messages}") - private int maxMessages; - - @Value("${queue.pubsub.executor_thread_pool_size:0}") - private int threadPoolSize; - - /** - * Refers to com.google.cloud.pubsub.v1.Publisher default executor configuration - */ - private static final int THREADS_PER_CPU = 5; - - private FixedExecutorProvider executorProvider; - - private CredentialsProvider credentialsProvider; - - @PostConstruct - private void init() throws IOException { - ServiceAccountCredentials credentials = ServiceAccountCredentials.fromStream( - new ByteArrayInputStream(serviceAccount.getBytes())); - credentialsProvider = FixedCredentialsProvider.create(credentials); - if (threadPoolSize == 0) { - threadPoolSize = THREADS_PER_CPU * Runtime.getRuntime().availableProcessors(); - } - executorProvider = FixedExecutorProvider - .create(ThingsBoardExecutors.newScheduledThreadPool(threadPoolSize, "pubsub-queue-executor")); - } - - @PreDestroy - private void destroy() { - if (executorProvider != null) { - executorProvider.getExecutor().shutdownNow(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java deleted file mode 100644 index 14aa67a4f0..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.pubsub; - -import jakarta.annotation.PostConstruct; -import lombok.Getter; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.queue.util.PropertyUtils; - -import java.util.Map; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='pubsub'") -public class TbPubSubSubscriptionSettings { - - @Value("${queue.pubsub.queue-properties.core:}") - private String coreProperties; - @Value("${queue.pubsub.queue-properties.rule-engine:}") - private String ruleEngineProperties; - @Value("${queue.pubsub.queue-properties.transport-api:}") - private String transportApiProperties; - @Value("${queue.pubsub.queue-properties.notifications:}") - private String notificationsProperties; - @Value("${queue.pubsub.queue-properties.js-executor:}") - private String jsExecutorProperties; - @Value("${queue.pubsub.queue-properties.version-control:}") - private String vcProperties; - @Value("${queue.pubsub.queue-properties.edge:}") - private String edgeProperties; - - @Getter - private Map coreSettings; - @Getter - private Map ruleEngineSettings; - @Getter - private Map transportApiSettings; - @Getter - private Map notificationsSettings; - @Getter - private Map jsExecutorSettings; - @Getter - private Map vcSettings; - @Getter - private Map edgeSettings; - - @PostConstruct - private void init() { - coreSettings = PropertyUtils.getProps(coreProperties); - ruleEngineSettings = PropertyUtils.getProps(ruleEngineProperties); - transportApiSettings = PropertyUtils.getProps(transportApiProperties); - notificationsSettings = PropertyUtils.getProps(notificationsProperties); - jsExecutorSettings = PropertyUtils.getProps(jsExecutorProperties); - vcSettings = PropertyUtils.getProps(vcProperties); - edgeSettings = PropertyUtils.getProps(edgeProperties); - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java deleted file mode 100644 index aad3c1c8cc..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.rabbitmq; - -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.thingsboard.server.queue.TbQueueAdmin; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeoutException; - -@Slf4j -@Deprecated(forRemoval = true, since = "3.9") // for removal in 4.0 -public class TbRabbitMqAdmin implements TbQueueAdmin { - - private final Channel channel; - private final Connection connection; - private final Map arguments; - - public TbRabbitMqAdmin(TbRabbitMqSettings rabbitMqSettings, Map arguments) { - this.arguments = arguments; - - try { - connection = rabbitMqSettings.getConnectionFactory().newConnection(); - } catch (IOException | TimeoutException e) { - log.error("Failed to create connection.", e); - throw new RuntimeException("Failed to create connection.", e); - } - - try { - channel = connection.createChannel(); - } catch (IOException e) { - log.error("Failed to create chanel.", e); - throw new RuntimeException("Failed to create chanel.", e); - } - } - - @Override - public void createTopicIfNotExists(String topic, String properties) { - Map arguments = this.arguments; - if (StringUtils.isNotBlank(properties)) { - arguments = new HashMap<>(arguments); - arguments.putAll(TbRabbitMqQueueArguments.getArgs(properties)); - } - try { - channel.queueDeclare(topic, false, false, false, arguments); - } catch (IOException e) { - log.error("Failed to bind queue: [{}]", topic, e); - } - } - - @Override - public void deleteTopic(String topic) { - try { - channel.queueDelete(topic); - } catch (IOException e) { - log.error("Failed to delete RabbitMq queue [{}].", topic); - } - } - - @Override - public void destroy() { - if (channel != null) { - try { - channel.close(); - } catch (IOException | TimeoutException e) { - log.error("Failed to close Chanel.", e); - } - } - if (connection != null) { - try { - connection.close(); - } catch (IOException e) { - log.error("Failed to close Connection.", e); - } - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java deleted file mode 100644 index e50afabb5d..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.rabbitmq; - -import com.google.gson.Gson; -import com.google.protobuf.InvalidProtocolBufferException; -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; -import com.rabbitmq.client.GetResponse; -import java.util.ArrayList; -import java.util.Collection; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueMsgDecoder; -import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; - -@Slf4j -public class TbRabbitMqConsumerTemplate extends AbstractTbQueueConsumerTemplate { - - private final Gson gson = new Gson(); - private final TbQueueAdmin admin; - private final TbQueueMsgDecoder decoder; - private final Channel channel; - private final Connection connection; - private final int maxPollMessages; - - private volatile Set queues; - - public TbRabbitMqConsumerTemplate(TbQueueAdmin admin, TbRabbitMqSettings rabbitMqSettings, String topic, TbQueueMsgDecoder decoder) { - super(topic); - this.admin = admin; - this.decoder = decoder; - this.maxPollMessages = rabbitMqSettings.getMaxPollMessages(); - try { - connection = rabbitMqSettings.getConnectionFactory().newConnection(); - } catch (IOException | TimeoutException e) { - log.error("Failed to create connection.", e); - throw new RuntimeException("Failed to create connection.", e); - } - try { - channel = connection.createChannel(); - } catch (IOException e) { - log.error("Failed to create chanel.", e); - throw new RuntimeException("Failed to create chanel.", e); - } - stopped = false; - } - - @Override - protected List doPoll(long durationInMillis) { - List result = queues.stream() - .map(queue -> { - List messages = new ArrayList<>(); - for (int i = 0; i < maxPollMessages; i++) { - GetResponse response = doQueuePoll(queue); - if (response == null) { - break; - } - messages.add(response); - } - return messages; - }) - .filter(r -> !r.isEmpty()) - .flatMap(Collection::stream) - .collect(Collectors.toList()); - if (result.size() > 0) { - return result; - } else { - return Collections.emptyList(); - } - } - - protected GetResponse doQueuePoll(String queue) { - try { - return channel.basicGet(queue, false); - } catch (IOException e) { - log.error("Failed to get messages from queue: [{}]", queue); - throw new RuntimeException("Failed to get messages from queue.", e); - } - } - - @Override - protected void doSubscribe(List topicNames) { - queues = partitions.stream() - .map(TopicPartitionInfo::getFullTopicName) - .collect(Collectors.toSet()); - queues.forEach(admin::createTopicIfNotExists); - } - - @Override - protected void doCommit() { - try { - channel.basicAck(0, true); - } catch (IOException e) { - log.error("Failed to ack messages.", e); - } - } - - @Override - protected void doUnsubscribe() { - if (channel != null) { - try { - channel.close(); - } catch (IOException | TimeoutException e) { - log.error("Failed to close the channel."); - } - } - if (connection != null) { - try { - connection.close(); - } catch (IOException e) { - log.error("Failed to close the connection."); - } - } - } - - public T decode(GetResponse message) throws InvalidProtocolBufferException { - DefaultTbQueueMsg msg = gson.fromJson(new String(message.getBody()), DefaultTbQueueMsg.class); - return decoder.decode(msg); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java deleted file mode 100644 index 696fcccce2..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqProducerTemplate.java +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.rabbitmq; - -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.gson.Gson; -import com.rabbitmq.client.AMQP; -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.io.IOException; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeoutException; - -@Slf4j -public class TbRabbitMqProducerTemplate implements TbQueueProducer { - private final String defaultTopic; - private final Gson gson = new Gson(); - private final TbQueueAdmin admin; - private final TbRabbitMqSettings rabbitMqSettings; - private final ListeningExecutorService producerExecutor; - private final Channel channel; - private final Connection connection; - - private final Set topics = ConcurrentHashMap.newKeySet(); - - public TbRabbitMqProducerTemplate(TbQueueAdmin admin, TbRabbitMqSettings rabbitMqSettings, String defaultTopic) { - this.admin = admin; - this.defaultTopic = defaultTopic; - this.rabbitMqSettings = rabbitMqSettings; - producerExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); - try { - connection = rabbitMqSettings.getConnectionFactory().newConnection(); - } catch (IOException | TimeoutException e) { - log.error("Failed to create connection.", e); - throw new RuntimeException("Failed to create connection.", e); - } - - try { - channel = connection.createChannel(); - } catch (IOException e) { - log.error("Failed to create chanel.", e); - throw new RuntimeException("Failed to create chanel.", e); - } - } - - @Override - public void init() { - - } - - @Override - public String getDefaultTopic() { - return defaultTopic; - } - - @Override - public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { - createTopicIfNotExist(tpi); - AMQP.BasicProperties properties = new AMQP.BasicProperties(); - try { - channel.basicPublish(rabbitMqSettings.getExchangeName(), tpi.getFullTopicName(), properties, gson.toJson(new DefaultTbQueueMsg(msg)).getBytes()); - if (callback != null) { - callback.onSuccess(null); - } - } catch (IOException e) { - log.error("Failed publish message: [{}].", msg, e); - if (callback != null) { - callback.onFailure(e); - } - } - } - - @Override - public void stop() { - if (producerExecutor != null) { - producerExecutor.shutdownNow(); - } - if (channel != null) { - try { - channel.close(); - } catch (IOException | TimeoutException e) { - log.error("Failed to close the channel."); - } - } - if (connection != null) { - try { - connection.close(); - } catch (IOException e) { - log.error("Failed to close the connection."); - } - } - } - - private void createTopicIfNotExist(TopicPartitionInfo tpi) { - if (topics.contains(tpi)) { - return; - } - admin.createTopicIfNotExists(tpi.getFullTopicName()); - topics.add(tpi); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java deleted file mode 100644 index 06613d5617..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.rabbitmq; - -import jakarta.annotation.PostConstruct; -import lombok.Getter; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.StringUtils; - -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Pattern; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'") -public class TbRabbitMqQueueArguments { - @Value("${queue.rabbitmq.queue-properties.core:}") - private String coreProperties; - @Value("${queue.rabbitmq.queue-properties.rule-engine:}") - private String ruleEngineProperties; - @Value("${queue.rabbitmq.queue-properties.transport-api:}") - private String transportApiProperties; - @Value("${queue.rabbitmq.queue-properties.notifications:}") - private String notificationsProperties; - @Value("${queue.rabbitmq.queue-properties.js-executor:}") - private String jsExecutorProperties; - @Value("${queue.rabbitmq.queue-properties.version-control:}") - private String vcProperties; - @Value("${queue.rabbitmq.queue-properties.edge:}") - private String edgeProperties; - - @Getter - private Map coreArgs; - @Getter - private Map ruleEngineArgs; - @Getter - private Map transportApiArgs; - @Getter - private Map notificationsArgs; - @Getter - private Map jsExecutorArgs; - @Getter - private Map vcArgs; - @Getter - private Map edgeArgs; - - @PostConstruct - private void init() { - coreArgs = getArgs(coreProperties); - ruleEngineArgs = getArgs(ruleEngineProperties); - transportApiArgs = getArgs(transportApiProperties); - notificationsArgs = getArgs(notificationsProperties); - jsExecutorArgs = getArgs(jsExecutorProperties); - vcArgs = getArgs(vcProperties); - edgeArgs = getArgs(edgeProperties); - } - - public static Map getArgs(String properties) { - Map configs = new HashMap<>(); - if (StringUtils.isNotEmpty(properties)) { - for (String property : properties.split(";")) { - int delimiterPosition = property.indexOf(":"); - String key = property.substring(0, delimiterPosition); - String strValue = property.substring(delimiterPosition + 1); - configs.put(key, getObjectValue(strValue)); - } - } - return configs; - } - - private static Object getObjectValue(String str) { - if (str.equalsIgnoreCase("true") || str.equalsIgnoreCase("false")) { - return Boolean.valueOf(str); - } else if (isNumeric(str)) { - return getNumericValue(str); - } - return str; - } - - private static Object getNumericValue(String str) { - if (str.contains(".")) { - return Double.valueOf(str); - } else { - return Long.valueOf(str); - } - } - - private static final Pattern PATTERN = Pattern.compile("-?\\d+(\\.\\d+)?"); - - private static boolean isNumeric(String strNum) { - if (strNum == null) { - return false; - } - return PATTERN.matcher(strNum).matches(); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java deleted file mode 100644 index c0a2912f23..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqSettings.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.rabbitmq; - -import com.rabbitmq.client.ConnectionFactory; -import jakarta.annotation.PostConstruct; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; - -@Slf4j -@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'") -@Component -@Data -public class TbRabbitMqSettings { - @Value("${queue.rabbitmq.exchange_name:}") - private String exchangeName; - @Value("${queue.rabbitmq.host:}") - private String host; - @Value("${queue.rabbitmq.port:}") - private int port; - @Value("${queue.rabbitmq.virtual_host:}") - private String virtualHost; - @Value("${queue.rabbitmq.username:}") - private String username; - @Value("${queue.rabbitmq.password:}") - private String password; - @Value("${queue.rabbitmq.automatic_recovery_enabled:}") - private boolean automaticRecoveryEnabled; - @Value("${queue.rabbitmq.connection_timeout:}") - private int connectionTimeout; - @Value("${queue.rabbitmq.handshake_timeout:}") - private int handshakeTimeout; - @Value("${queue.rabbitmq.max_poll_messages:1}") - private int maxPollMessages; - - private ConnectionFactory connectionFactory; - - @PostConstruct - private void init() { - connectionFactory = new ConnectionFactory(); - connectionFactory.setHost(host); - connectionFactory.setPort(port); - connectionFactory.setVirtualHost(virtualHost); - connectionFactory.setUsername(username); - connectionFactory.setPassword(password); - connectionFactory.setAutomaticRecoveryEnabled(automaticRecoveryEnabled); - connectionFactory.setConnectionTimeout(connectionTimeout); - connectionFactory.setHandshakeTimeout(handshakeTimeout); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java deleted file mode 100644 index 8d13a543d3..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/AwsSqsTbQueueMsgMetadata.java +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.sqs; - -import com.amazonaws.http.SdkHttpMetadata; -import lombok.AllArgsConstructor; -import lombok.Data; -import org.thingsboard.server.queue.TbQueueMsgMetadata; - -@Data -@AllArgsConstructor -public class AwsSqsTbQueueMsgMetadata implements TbQueueMsgMetadata { - - private final SdkHttpMetadata metadata; -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java deleted file mode 100644 index 6aa1ec4dad..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.sqs; - -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; -import com.amazonaws.services.sqs.AmazonSQS; -import com.amazonaws.services.sqs.AmazonSQSClientBuilder; -import com.amazonaws.services.sqs.model.CreateQueueRequest; -import com.amazonaws.services.sqs.model.GetQueueUrlResult; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.util.PropertyUtils; - -import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.function.Function; -import java.util.stream.Collectors; - -@Slf4j -@Deprecated(forRemoval = true, since = "3.9") // for removal in 4.0 -public class TbAwsSqsAdmin implements TbQueueAdmin { - - private final Map attributes; - private final AmazonSQS sqsClient; - private final Map queues; - @Getter - private final ExecutorService producerExecutor; - - public TbAwsSqsAdmin(TbAwsSqsSettings sqsSettings, Map attributes) { - this.attributes = attributes; - - AWSCredentialsProvider credentialsProvider; - if (sqsSettings.getUseDefaultCredentialProviderChain()) { - credentialsProvider = new DefaultAWSCredentialsProviderChain(); - } else { - AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); - credentialsProvider = new AWSStaticCredentialsProvider(awsCredentials); - } - producerExecutor = ThingsBoardExecutors.newWorkStealingPool(sqsSettings.getThreadPoolSize(), "aws-sqs-queue-executor"); - - sqsClient = AmazonSQSClientBuilder.standard() - .withCredentials(credentialsProvider) - .withRegion(sqsSettings.getRegion()) - .build(); - - queues = sqsClient - .listQueues() - .getQueueUrls() - .stream() - .map(this::getQueueNameFromUrl) - .collect(Collectors.toMap(this::convertTopicToQueueName, Function.identity())); - } - - @Override - public void createTopicIfNotExists(String topic, String properties) { - String queueName = convertTopicToQueueName(topic); - if (queues.containsKey(queueName)) { - return; - } - Map attributes = PropertyUtils.getProps(this.attributes, properties, TbAwsSqsQueueAttributes::toConfigs); - final CreateQueueRequest createQueueRequest = new CreateQueueRequest(queueName).withAttributes(attributes); - String queueUrl = sqsClient.createQueue(createQueueRequest).getQueueUrl(); - queues.put(getQueueNameFromUrl(queueUrl), queueUrl); - } - - private String convertTopicToQueueName(String topic) { - return topic.replaceAll("\\.", "_") + ".fifo"; - } - - @Override - public void deleteTopic(String topic) { - String queueName = convertTopicToQueueName(topic); - if (queues.containsKey(queueName)) { - sqsClient.deleteQueue(queues.get(queueName)); - } else { - GetQueueUrlResult queueUrl = sqsClient.getQueueUrl(queueName); - if (queueUrl != null) { - sqsClient.deleteQueue(queueUrl.getQueueUrl()); - } else { - log.warn("Aws SQS queue [{}] does not exist!", queueName); - } - } - } - - private String getQueueNameFromUrl(String queueUrl) { - int delimiterIndex = queueUrl.lastIndexOf("/"); - return queueUrl.substring(delimiterIndex + 1); - } - - @Override - public void destroy() { - if (sqsClient != null) { - sqsClient.shutdown(); - } - if (producerExecutor != null) { - producerExecutor.shutdownNow(); - } - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java deleted file mode 100644 index a3087e9d11..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java +++ /dev/null @@ -1,188 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.sqs; - -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; -import com.amazonaws.services.sqs.AmazonSQS; -import com.amazonaws.services.sqs.AmazonSQSClientBuilder; -import com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry; -import com.amazonaws.services.sqs.model.Message; -import com.amazonaws.services.sqs.model.ReceiveMessageRequest; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.gson.Gson; -import com.google.protobuf.InvalidProtocolBufferException; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.CollectionUtils; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueMsgDecoder; -import org.thingsboard.server.queue.common.AbstractParallelTbQueueConsumerTemplate; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -@Slf4j -public class TbAwsSqsConsumerTemplate extends AbstractParallelTbQueueConsumerTemplate { - - private static final int MAX_NUM_MSGS = 10; - - private final Gson gson = new Gson(); - private final TbQueueAdmin admin; - private final AmazonSQS sqsClient; - private final TbQueueMsgDecoder decoder; - private final TbAwsSqsSettings sqsSettings; - - private final List pendingMessages = new CopyOnWriteArrayList<>(); - private volatile Set queueUrls; - - public TbAwsSqsConsumerTemplate(TbQueueAdmin admin, TbAwsSqsSettings sqsSettings, String topic, TbQueueMsgDecoder decoder) { - super(topic); - this.admin = admin; - this.decoder = decoder; - this.sqsSettings = sqsSettings; - - AWSCredentialsProvider credentialsProvider; - if (sqsSettings.getUseDefaultCredentialProviderChain()) { - credentialsProvider = new DefaultAWSCredentialsProviderChain(); - } else { - AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); - credentialsProvider = new AWSStaticCredentialsProvider(awsCredentials); - } - - sqsClient = AmazonSQSClientBuilder.standard() - .withCredentials(credentialsProvider) - .withRegion(sqsSettings.getRegion()) - .build(); - - } - - @Override - protected void doSubscribe(List topicNames) { - queueUrls = topicNames.stream().map(this::getQueueUrl).collect(Collectors.toSet()); - initNewExecutor(queueUrls.size() * sqsSettings.getThreadsPerTopic() + 1); - } - - @Override - protected List doPoll(long durationInMillis) { - int duration = (int) TimeUnit.MILLISECONDS.toSeconds(durationInMillis); - List>> futureList = queueUrls - .stream() - .map(url -> poll(url, duration)) - .collect(Collectors.toList()); - ListenableFuture>> futureResult = Futures.allAsList(futureList); - try { - return futureResult.get().stream() - .flatMap(List::stream) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - } catch (InterruptedException | ExecutionException e) { - if (stopped) { - log.info("[{}] Aws SQS consumer is stopped.", getTopic()); - } else { - log.error("Failed to pool messages.", e); - } - return Collections.emptyList(); - } - } - - @Override - public T decode(Message message) throws InvalidProtocolBufferException { - DefaultTbQueueMsg msg = gson.fromJson(message.getBody(), DefaultTbQueueMsg.class); - return decoder.decode(msg); - } - - @Override - protected void doCommit() { - pendingMessages.forEach(msg -> - consumerExecutor.submit(() -> { - List entries = msg.getMessages() - .stream() - .map(message -> new DeleteMessageBatchRequestEntry(message.getMessageId(), message.getReceiptHandle())) - .collect(Collectors.toList()); - sqsClient.deleteMessageBatch(msg.getUrl(), entries); - })); - pendingMessages.clear(); - } - - @Override - protected void doUnsubscribe() { - stopped = true; - if (sqsClient != null) { - sqsClient.shutdown(); - } - shutdownExecutor(); - } - - private ListenableFuture> poll(String url, int waitTimeSeconds) { - List>> result = new ArrayList<>(); - - for (int i = 0; i < sqsSettings.getThreadsPerTopic(); i++) { - result.add(consumerExecutor.submit(() -> { - ReceiveMessageRequest request = new ReceiveMessageRequest(); - request - .withWaitTimeSeconds(waitTimeSeconds) - .withQueueUrl(url) - .withMaxNumberOfMessages(MAX_NUM_MSGS); - return sqsClient.receiveMessage(request).getMessages(); - })); - } - return Futures.transform(Futures.allAsList(result), list -> { - if (!CollectionUtils.isEmpty(list)) { - return list.stream() - .flatMap(messageList -> { - if (!messageList.isEmpty()) { - this.pendingMessages.add(new AwsSqsMsgWrapper(url, messageList)); - return messageList.stream(); - } - return Stream.empty(); - }) - .collect(Collectors.toList()); - } - return Collections.emptyList(); - }, consumerExecutor); - } - - @Data - private static class AwsSqsMsgWrapper { - private final String url; - private final List messages; - - public AwsSqsMsgWrapper(String url, List messages) { - this.url = url; - this.messages = messages; - } - } - - private String getQueueUrl(String topic) { - admin.createTopicIfNotExists(topic); - return sqsClient.getQueueUrl(topic.replaceAll("\\.", "_") + ".fifo").getQueueUrl(); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java deleted file mode 100644 index d8d4a5d2e8..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java +++ /dev/null @@ -1,117 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.sqs; - -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; -import com.amazonaws.handlers.AsyncHandler; -import com.amazonaws.services.sqs.AmazonSQSAsync; -import com.amazonaws.services.sqs.AmazonSQSAsyncClientBuilder; -import com.amazonaws.services.sqs.model.SendMessageRequest; -import com.amazonaws.services.sqs.model.SendMessageResult; -import com.google.gson.Gson; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsg; -import org.thingsboard.server.queue.TbQueueProducer; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; - -@Slf4j -public class TbAwsSqsProducerTemplate implements TbQueueProducer { - private final String defaultTopic; - private final AmazonSQSAsync sqsClient; - private final Gson gson = new Gson(); - private final Map queueUrlMap = new ConcurrentHashMap<>(); - private final TbAwsSqsAdmin admin; - - public TbAwsSqsProducerTemplate(TbQueueAdmin admin, TbAwsSqsSettings sqsSettings, String defaultTopic) { - this.admin = (TbAwsSqsAdmin) admin; - this.defaultTopic = defaultTopic; - - AWSCredentialsProvider credentialsProvider; - if (sqsSettings.getUseDefaultCredentialProviderChain()) { - credentialsProvider = new DefaultAWSCredentialsProviderChain(); - } else { - AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); - credentialsProvider = new AWSStaticCredentialsProvider(awsCredentials); - } - - sqsClient = AmazonSQSAsyncClientBuilder.standard() - .withCredentials(credentialsProvider) - .withRegion(sqsSettings.getRegion()) - .withExecutorFactory(this.admin::getProducerExecutor) - .build(); - } - - @Override - public void init() { - - } - - @Override - public String getDefaultTopic() { - return defaultTopic; - } - - @Override - public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { - SendMessageRequest sendMsgRequest = new SendMessageRequest(); - sendMsgRequest.withQueueUrl(getQueueUrl(tpi.getFullTopicName())); - sendMsgRequest.withMessageBody(gson.toJson(new DefaultTbQueueMsg(msg))); - - String sqsMsgId = UUID.randomUUID().toString(); - sendMsgRequest.withMessageGroupId(sqsMsgId); - sendMsgRequest.withMessageDeduplicationId(sqsMsgId); - - sqsClient.sendMessageAsync(sendMsgRequest, new AsyncHandler() { - @Override public void onError(Exception e) { - if (callback != null) { - callback.onFailure(e); - } - } - - @Override public void onSuccess(SendMessageRequest request, - SendMessageResult sendMessageResult) { - if (callback != null) { - callback.onSuccess(new AwsSqsTbQueueMsgMetadata(sendMessageResult.getSdkHttpMetadata())); - } - } - }); - } - - @Override - public void stop() { - if (sqsClient != null) { - sqsClient.shutdown(); - } - } - - private String getQueueUrl(String topic) { - return queueUrlMap.computeIfAbsent(topic, k -> { - admin.createTopicIfNotExists(topic); - return sqsClient.getQueueUrl(topic.replaceAll("\\.", "_") + ".fifo").getQueueUrl(); - }); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java deleted file mode 100644 index 2c1b8793eb..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java +++ /dev/null @@ -1,105 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.sqs; - -import com.amazonaws.services.sqs.model.QueueAttributeName; -import jakarta.annotation.PostConstruct; -import lombok.Getter; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.StringUtils; - -import java.util.HashMap; -import java.util.Map; - -@Component -@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs'") -public class TbAwsSqsQueueAttributes { - @Value("${queue.aws-sqs.queue-properties.core:}") - private String coreProperties; - @Value("${queue.aws-sqs.queue-properties.rule-engine:}") - private String ruleEngineProperties; - @Value("${queue.aws-sqs.queue-properties.transport-api:}") - private String transportApiProperties; - @Value("${queue.aws-sqs.queue-properties.notifications:}") - private String notificationsProperties; - @Value("${queue.aws-sqs.queue-properties.js-executor:}") - private String jsExecutorProperties; - @Value("${queue.aws-sqs.queue-properties.ota-updates:}") - private String otaProperties; - @Value("${queue.aws-sqs.queue-properties.version-control:}") - private String vcProperties; - @Value("${queue.aws-sqs.queue-properties.edge:}") - private String edgeProperties; - - @Getter - private Map coreAttributes; - @Getter - private Map ruleEngineAttributes; - @Getter - private Map transportApiAttributes; - @Getter - private Map notificationsAttributes; - @Getter - private Map jsExecutorAttributes; - @Getter - private Map otaAttributes; - @Getter - private Map vcAttributes; - @Getter - private Map edgeAttributes; - - private final Map defaultAttributes = new HashMap<>(); - - @PostConstruct - private void init() { - defaultAttributes.put(QueueAttributeName.FifoQueue.toString(), "true"); - - coreAttributes = getConfigs(coreProperties); - ruleEngineAttributes = getConfigs(ruleEngineProperties); - transportApiAttributes = getConfigs(transportApiProperties); - notificationsAttributes = getConfigs(notificationsProperties); - jsExecutorAttributes = getConfigs(jsExecutorProperties); - otaAttributes = getConfigs(otaProperties); - vcAttributes = getConfigs(vcProperties); - edgeAttributes = getConfigs(edgeProperties); - } - - private Map getConfigs(String properties) { - Map configs = new HashMap<>(defaultAttributes); - configs.putAll(toConfigs(properties)); - return configs; - } - - public static Map toConfigs(String properties) { - Map configs = new HashMap<>(); - if (StringUtils.isNotEmpty(properties)) { - for (String property : properties.split(";")) { - int delimiterPosition = property.indexOf(":"); - String key = property.substring(0, delimiterPosition); - String value = property.substring(delimiterPosition + 1); - validateAttributeName(key); - configs.put(key, value); - } - } - return configs; - } - - private static void validateAttributeName(String key) { - QueueAttributeName.fromValue(key); - } -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java deleted file mode 100644 index 122d5f0780..0000000000 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.sqs; - -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Component; - -@Slf4j -@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs'") -@Component -@Data -public class TbAwsSqsSettings { - - @Value("${queue.aws_sqs.use_default_credential_provider_chain}") - private Boolean useDefaultCredentialProviderChain; - - @Value("${queue.aws_sqs.access_key_id}") - private String accessKeyId; - - @Value("${queue.aws_sqs.secret_access_key}") - private String secretAccessKey; - - @Value("${queue.aws_sqs.region}") - private String region; - - @Value("${queue.aws_sqs.threads_per_topic}") - private int threadsPerTopic; - - @Value("${queue.aws_sqs.producer_thread_pool_size:50}") - private int threadPoolSize; - -} diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplateTest.java deleted file mode 100644 index 2d6a50cea4..0000000000 --- a/common/queue/src/test/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplateTest.java +++ /dev/null @@ -1,128 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.queue.rabbitmq; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; -import com.rabbitmq.client.ConnectionFactory; -import com.rabbitmq.client.GetResponse; -import java.nio.charset.StandardCharsets; -import java.util.Set; -import java.util.UUID; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueMsgDecoder; -import org.thingsboard.server.queue.common.DefaultTbQueueMsg; - -@ExtendWith(MockitoExtension.class) -class TbRabbitMqConsumerTemplateTest { - - private static final String TOPIC = "some-topic"; - - @Mock - private TbQueueAdmin admin; - - @Mock - private ConnectionFactory connectionFactory; - - @Mock - private TbQueueMsgDecoder decoder; - - @Mock - private Connection connection; - - @Mock - private Channel channel; - - @Mock - private TopicPartitionInfo partition; - - @Mock - private GetResponse getResponse; - - private TbRabbitMqConsumerTemplate consumer; - - private void setUpConsumerWithMaxPollMessages(int maxPollMessages) throws Exception { - when(connectionFactory.newConnection()).thenReturn(connection); - when(connection.createChannel()).thenReturn(channel); - TbRabbitMqSettings settings = new TbRabbitMqSettings(); - settings.setMaxPollMessages(maxPollMessages); - settings.setConnectionFactory(connectionFactory); - - consumer = new TbRabbitMqConsumerTemplate<>(admin, settings, TOPIC, decoder); - when(partition.getFullTopicName()).thenReturn(TOPIC); - consumer.subscribe(Set.of(partition)); - } - - @Test - void pollWithMax5PollMessagesReturnsEmptyListIfNoMessages() throws Exception { - setUpConsumerWithMaxPollMessages(5); - when(channel.basicGet(anyString(), anyBoolean())).thenReturn(null); - - assertThat(consumer.poll(0L)).isEmpty(); - - verify(channel).basicGet(anyString(), anyBoolean()); - } - - @Test - void pollWithMax5PollMessagesReturns5MessagesIfQueueContains5() throws Exception { - setUpConsumerWithMaxPollMessages(5); - when(getResponse.getBody()).thenReturn(newMessageBody()); - when(channel.basicGet(anyString(), anyBoolean())).thenReturn(getResponse); - - assertThat(consumer.poll(0L)).hasSize(5); - - verify(channel, times(5)).basicGet(anyString(), anyBoolean()); - } - - @Test - void pollWithMax1PollMessageReturns1MessageIfQueueContainsMore() throws Exception { - setUpConsumerWithMaxPollMessages(1); - when(getResponse.getBody()).thenReturn(newMessageBody()); - when(channel.basicGet(anyString(), anyBoolean())).thenReturn(getResponse); - - assertThat(consumer.poll(0L)).hasSize(1); - - verify(channel).basicGet(anyString(), anyBoolean()); - } - - @Test - void pollWithMax3PollMessagesReturns2MessagesIfQueueContains2() throws Exception { - setUpConsumerWithMaxPollMessages(3); - when(getResponse.getBody()).thenReturn(newMessageBody()); - when(channel.basicGet(anyString(), anyBoolean())).thenReturn(getResponse, getResponse, null); - - assertThat(consumer.poll(0L)).hasSize(2); - - verify(channel, times(3)).basicGet(anyString(), anyBoolean()); - } - - private byte[] newMessageBody() { - return ("{\"key\": \"" + UUID.randomUUID() + "\"}").getBytes(StandardCharsets.UTF_8); - } - -} \ No newline at end of file diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 2b7e2a51dc..e669016514 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -94,6 +94,7 @@ import org.thingsboard.server.transport.mqtt.util.sparkplug.SparkplugTopic; import javax.net.ssl.SSLPeerUnverifiedException; import java.io.IOException; import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; import java.security.cert.Certificate; import java.security.cert.X509Certificate; import java.util.ArrayList; @@ -109,7 +110,6 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static com.amazonaws.util.StringUtils.UTF8; import static io.netty.handler.codec.mqtt.MqttMessageType.CONNECT; import static io.netty.handler.codec.mqtt.MqttMessageType.PINGRESP; import static io.netty.handler.codec.mqtt.MqttMessageType.SUBACK; @@ -606,7 +606,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement } private void getOtaPackageCallback(ChannelHandlerContext ctx, MqttPublishMessage mqttMsg, int msgId, Matcher fwMatcher, OtaPackageType type) { - String payload = mqttMsg.content().toString(UTF8); + String payload = mqttMsg.content().toString(StandardCharsets.UTF_8); int chunkSize = StringUtils.isNotEmpty(payload) ? Integer.parseInt(payload) : 0; String requestId = fwMatcher.group("requestId"); int chunk = Integer.parseInt(fwMatcher.group("chunk")); diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index 7c79eda0d1..f28df78d1d 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -14,7 +14,7 @@ # limitations under the License. # -queue_type: "TB_QUEUE_TYPE" #kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) +queue_type: "TB_QUEUE_TYPE" #kafka (Apache Kafka) queue_prefix: "TB_QUEUE_PREFIX" request_topic: "REMOTE_JS_EVAL_REQUEST_TOPIC" http_port: "HTTP_PORT" # /livenessProbe @@ -55,32 +55,6 @@ kafka: username: "TB_QUEUE_KAFKA_CONFLUENT_USERNAME" password: "TB_QUEUE_KAFKA_CONFLUENT_PASSWORD" -pubsub: - project_id: "TB_QUEUE_PUBSUB_PROJECT_ID" - service_account: "TB_QUEUE_PUBSUB_SERVICE_ACCOUNT" - queue_properties: "TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES" - -aws_sqs: - access_key_id: "TB_QUEUE_AWS_SQS_ACCESS_KEY_ID" - secret_access_key: "TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY" - region: "TB_QUEUE_AWS_SQS_REGION" - queue_properties: "TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES" - -rabbitmq: - host: "TB_QUEUE_RABBIT_MQ_HOST" - port: "TB_QUEUE_RABBIT_MQ_PORT" - virtual_host: "TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST" - username: "TB_QUEUE_RABBIT_MQ_USERNAME" - password: "TB_QUEUE_RABBIT_MQ_PASSWORD" - queue_properties: "TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES" - -service_bus: - namespace_name: "TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME" - sas_key_name: "TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME" - sas_key: "TB_QUEUE_SERVICE_BUS_SAS_KEY" - max_messages: "TB_QUEUE_SERVICE_BUS_MAX_MESSAGES" - queue_properties: "TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES" - logger: level: "LOGGER_LEVEL" path: "LOG_FOLDER" diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index f5bde183e4..dda3ef02c7 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -44,23 +44,6 @@ kafka: sasl: mechanism: "PLAIN" -pubsub: - queue_properties: "ackDeadlineInSec:30;messageRetentionInSec:604800" - -aws_sqs: - queue_properties: "VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800" - -rabbitmq: - host: "localhost" - port: "5672" - virtual_host: "/" - username: "admin" - password: "password" - queue_properties: "x-max-length-bytes:1048576000;x-message-ttl:604800000" - -service_bus: - queue_properties: "lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800" - logger: level: "info" path: "logs" diff --git a/msa/js-executor/queue/awsSqsTemplate.ts b/msa/js-executor/queue/awsSqsTemplate.ts deleted file mode 100644 index 3edb0db1f4..0000000000 --- a/msa/js-executor/queue/awsSqsTemplate.ts +++ /dev/null @@ -1,198 +0,0 @@ -/// -/// Copyright © 2016-2024 The Thingsboard Authors -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// - -import config from 'config'; -import { _logger } from '../config/logger'; -import { JsInvokeMessageProcessor } from '../api/jsInvokeMessageProcessor' -import { IQueue } from './queue.models'; -import { - CreateQueueCommand, - CreateQueueRequest, - DeleteMessageBatchCommand, - DeleteMessageBatchRequest, - DeleteMessageBatchRequestEntry, - ListQueuesCommand, - ListQueuesResult, - ReceiveMessageCommand, - ReceiveMessageRequest, - ReceiveMessageResult, - SendMessageCommand, - SendMessageRequest, - SQSClient -} from '@aws-sdk/client-sqs'; -import uuid from 'uuid-random'; - -export class AwsSqsTemplate implements IQueue { - - private logger = _logger(`awsSqsTemplate`); - private queuePrefix: string = config.get('queue_prefix'); - private requestTopic: string = this.queuePrefix ? this.queuePrefix + "." + config.get('request_topic') : config.get('request_topic'); - private accessKeyId: string = config.get('aws_sqs.access_key_id'); - private secretAccessKey: string = config.get('aws_sqs.secret_access_key'); - private region: string = config.get('aws_sqs.region'); - private queueProperties: string = config.get('aws_sqs.queue_properties'); - private pollInterval = Number(config.get('js.response_poll_interval')); - - private sqsClient: SQSClient; - private requestQueueURL: string - private queueUrls = new Map(); - private queueAttributes: { [n: string]: string } = { - FifoQueue: 'true' - }; - private timer: NodeJS.Timeout; - - name = 'AWS SQS'; - - constructor() { - } - - async init() { - this.sqsClient = new SQSClient({ - apiVersion: '2012-11-05', - credentials: { - accessKeyId: this.accessKeyId, - secretAccessKey: this.secretAccessKey - }, - region: this.region - }); - - const queues = await this.getQueues(); - - if (queues.QueueUrls) { - queues.QueueUrls.forEach(queueUrl => { - const delimiterPosition = queueUrl.lastIndexOf('/'); - const queueName = queueUrl.substring(delimiterPosition + 1); - this.queueUrls.set(queueName, queueUrl); - }); - } - - this.parseQueueProperties(); - - this.requestQueueURL = this.queueUrls.get(AwsSqsTemplate.topicToSqsQueueName(this.requestTopic)) || ''; - if (!this.requestQueueURL) { - this.requestQueueURL = await this.createQueue(this.requestTopic); - } - - const messageProcessor = new JsInvokeMessageProcessor(this); - - const params: ReceiveMessageRequest = { - MaxNumberOfMessages: 10, - QueueUrl: this.requestQueueURL, - WaitTimeSeconds: Math.ceil(this.pollInterval / 10) - }; - this.timer = setTimeout(() => {this.getAndProcessMessage(messageProcessor, params)}, this.pollInterval); - } - - private async getAndProcessMessage(messageProcessor: JsInvokeMessageProcessor, params: ReceiveMessageRequest) { - const messagesResponse: ReceiveMessageResult = await this.sqsClient.send(new ReceiveMessageCommand(params)); - const messages = messagesResponse.Messages; - - if (messages && messages.length > 0) { - const entries: DeleteMessageBatchRequestEntry[] = []; - - messages.forEach(message => { - entries.push({ - Id: message.MessageId, - ReceiptHandle: message.ReceiptHandle - }); - messageProcessor.onJsInvokeMessage(JSON.parse(message.Body || '')); - }); - - const deleteBatch: DeleteMessageBatchRequest = { - QueueUrl: this.requestQueueURL, - Entries: entries - }; - try { - await this.sqsClient.send(new DeleteMessageBatchCommand(deleteBatch)) - } catch (err: any) { - this.logger.error("Failed to delete messages from queue.", err.message); - } - } - this.timer = setTimeout(() => {this.getAndProcessMessage(messageProcessor, params)}, this.pollInterval); - } - - async send(responseTopic: string, msgKey: string, rawResponse: Buffer, headers: any): Promise { - let msgBody = JSON.stringify( - { - key: msgKey, - data: [...rawResponse], - headers: headers - }); - - let responseQueueUrl = this.queueUrls.get(AwsSqsTemplate.topicToSqsQueueName(responseTopic)); - - if (!responseQueueUrl) { - responseQueueUrl = await this.createQueue(responseTopic); - this.queueUrls.set(responseTopic, responseQueueUrl); - } - - let msgId = uuid(); - - let params: SendMessageRequest = { - MessageBody: msgBody, - QueueUrl: responseQueueUrl, - MessageGroupId: msgId, - MessageDeduplicationId: msgId - }; - - return this.sqsClient.send(new SendMessageCommand(params)) - } - - private async getQueues(): Promise { - return this.sqsClient.send(new ListQueuesCommand({})); - } - - private parseQueueProperties() { - const props = this.queueProperties.split(';'); - props.forEach(p => { - const delimiterPosition = p.indexOf(':'); - this.queueAttributes[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1); - }); - } - - private static topicToSqsQueueName(topic: string): string { - return topic.replace(/\./g, '_') + '.fifo'; - } - - private async createQueue(topic: string): Promise { - let queueName = AwsSqsTemplate.topicToSqsQueueName(topic); - let queueParams: CreateQueueRequest = { - QueueName: queueName, - Attributes: this.queueAttributes - }; - - const result = await this.sqsClient.send(new CreateQueueCommand(queueParams)); - return result.QueueUrl || ''; - } - - async destroy(): Promise { - this.logger.info('Stopping AWS SQS resources...'); - clearTimeout(this.timer); - if (this.sqsClient) { - this.logger.info('Stopping AWS SQS client...'); - try { - const _sqsClient = this.sqsClient; - // @ts-ignore - delete this.sqsClient; - _sqsClient.destroy(); - this.logger.info('AWS SQS client stopped.'); - } catch (e: any) { - this.logger.info('AWS SQS client stop error.'); - } - } - this.logger.info('AWS SQS resources stopped.') - } -} diff --git a/msa/js-executor/queue/kafkaTemplate.ts b/msa/js-executor/queue/kafkaTemplate.ts index 659d1fb8b2..309edca61d 100644 --- a/msa/js-executor/queue/kafkaTemplate.ts +++ b/msa/js-executor/queue/kafkaTemplate.ts @@ -35,6 +35,7 @@ import { KeyObject } from 'tls'; import process, { exit, kill } from 'process'; +// TODO: remove dependencies for other queue types export class KafkaTemplate implements IQueue { private logger = _logger(`kafkaTemplate`); diff --git a/msa/js-executor/queue/pubSubTemplate.ts b/msa/js-executor/queue/pubSubTemplate.ts deleted file mode 100644 index 537d2e71b1..0000000000 --- a/msa/js-executor/queue/pubSubTemplate.ts +++ /dev/null @@ -1,162 +0,0 @@ -/// -/// Copyright © 2016-2024 The Thingsboard Authors -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// - -import config from 'config'; -import { _logger } from '../config/logger'; -import { JsInvokeMessageProcessor } from '../api/jsInvokeMessageProcessor' -import { PubSub } from '@google-cloud/pubsub'; -import { IQueue } from './queue.models'; -import { Message } from '@google-cloud/pubsub/build/src/subscriber'; - -export class PubSubTemplate implements IQueue { - - private logger = _logger(`pubSubTemplate`); - private projectId: string = config.get('pubsub.project_id'); - private credentials = JSON.parse(config.get('pubsub.service_account')); - private queuePrefix: string = config.get('queue_prefix'); - private requestTopic: string = this.queuePrefix ? this.queuePrefix + "." + config.get('request_topic') : config.get('request_topic'); - private queueProperties: string = config.get('pubsub.queue_properties'); - - private pubSubClient: PubSub; - private queueProps: { [n: string]: string } = {}; - private topics: string[] = []; - private subscriptions: string[] = []; - - name = 'Pub/Sub'; - - constructor() { - } - - async init() { - this.pubSubClient = new PubSub({ - projectId: this.projectId, - credentials: this.credentials - }); - - this.parseQueueProperties(); - - const topicList = await this.pubSubClient.getTopics(); - - if (topicList) { - topicList[0].forEach(topic => { - this.topics.push(PubSubTemplate.getName(topic.name)); - }); - } - - const subscriptionList = await this.pubSubClient.getSubscriptions(); - - if (subscriptionList) { - topicList[0].forEach(sub => { - this.subscriptions.push(PubSubTemplate.getName(sub.name)); - }); - } - - if (!(this.subscriptions.includes(this.requestTopic) && this.topics.includes(this.requestTopic))) { - await this.createTopic(this.requestTopic); - await this.createSubscription(this.requestTopic); - } - - const subscription = this.pubSubClient.subscription(this.requestTopic); - - const messageProcessor = new JsInvokeMessageProcessor(this); - - const messageHandler = (message: Message) => { - messageProcessor.onJsInvokeMessage(JSON.parse(message.data.toString('utf8'))); - message.ack(); - }; - - subscription.on('message', messageHandler); - } - - async send(responseTopic: string, msgKey: string, rawResponse: Buffer, headers: any): Promise { - if (!(this.subscriptions.includes(responseTopic) && this.topics.includes(this.requestTopic))) { - await this.createTopic(this.requestTopic); - await this.createSubscription(this.requestTopic); - } - - let data = JSON.stringify( - { - key: msgKey, - data: [...rawResponse], - headers: headers - }); - let dataBuffer = Buffer.from(data); - return this.pubSubClient.topic(responseTopic).publishMessage({data: dataBuffer}); - } - - private parseQueueProperties() { - const props = this.queueProperties.split(';'); - props.forEach(p => { - const delimiterPosition = p.indexOf(':'); - this.queueProps[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1); - }); - } - - private static getName(fullName: string): string { - const delimiterPosition = fullName.lastIndexOf('/'); - return fullName.substring(delimiterPosition + 1); - } - - private async createTopic(topic: string) { - if (!this.topics.includes(topic)) { - try { - await this.pubSubClient.createTopic(topic); - this.logger.info('Created new Pub/Sub topic: %s', topic); - } catch (e) { - this.logger.info('Pub/Sub topic already exists'); - } - this.topics.push(topic); - } - } - - private async createSubscription(topic: string) { - if (!this.subscriptions.includes(topic)) { - try { - await this.pubSubClient.createSubscription(topic, topic, { - topic: topic, - name: topic, - ackDeadlineSeconds: Number(this.queueProps['ackDeadlineInSec']), - messageRetentionDuration: { - seconds: this.queueProps['messageRetentionInSec'] - } - }); - this.logger.info('Created new Pub/Sub subscription: %s', topic); - } catch (e) { - this.logger.info('Pub/Sub subscription already exists.'); - } - - this.subscriptions.push(topic); - } - } - - async destroy(): Promise { - this.logger.info('Stopping Pub/Sub resources...'); - if (this.pubSubClient) { - this.logger.info('Stopping Pub/Sub client...'); - try { - const _pubSubClient = this.pubSubClient; - // @ts-ignore - delete this.pubSubClient; - await _pubSubClient.close(); - this.logger.info('Pub/Sub client stopped.'); - } catch (e) { - this.logger.info('Pub/Sub client stop error.'); - } - } - this.logger.info('Pub/Sub resources stopped.'); - } -} - diff --git a/msa/js-executor/queue/rabbitmqTemplate.ts b/msa/js-executor/queue/rabbitmqTemplate.ts deleted file mode 100644 index 9219afedc9..0000000000 --- a/msa/js-executor/queue/rabbitmqTemplate.ts +++ /dev/null @@ -1,128 +0,0 @@ -/// -/// Copyright © 2016-2024 The Thingsboard Authors -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// - -import config from 'config'; -import { _logger } from '../config/logger'; -import { JsInvokeMessageProcessor } from '../api/jsInvokeMessageProcessor' -import { IQueue } from './queue.models'; -import amqp, { ConfirmChannel, Connection } from 'amqplib'; -import { Options, Replies } from 'amqplib/properties'; - -export class RabbitMqTemplate implements IQueue { - - private logger = _logger(`rabbitmqTemplate`); - private queuePrefix: string = config.get('queue_prefix'); - private requestTopic: string = this.queuePrefix ? this.queuePrefix + "." + config.get('request_topic') : config.get('request_topic'); - private host = config.get('rabbitmq.host'); - private port = config.get('rabbitmq.port'); - private vhost = config.get('rabbitmq.virtual_host'); - private username = config.get('rabbitmq.username'); - private password = config.get('rabbitmq.password'); - private queueProperties: string = config.get('rabbitmq.queue_properties'); - - private queueOptions: Options.AssertQueue = { - durable: false, - exclusive: false, - autoDelete: false - }; - private connection: Connection; - private channel: ConfirmChannel; - private topics: string[] = []; - - name = 'RabbitMQ'; - - constructor() { - } - - async init(): Promise { - const url = `amqp://${this.username}:${this.password}@${this.host}:${this.port}${this.vhost}`; - this.connection = await amqp.connect(url); - this.channel = await this.connection.createConfirmChannel(); - - this.parseQueueProperties(); - - await this.createQueue(this.requestTopic); - - const messageProcessor = new JsInvokeMessageProcessor(this); - - await this.channel.consume(this.requestTopic, (message) => { - if (message) { - messageProcessor.onJsInvokeMessage(JSON.parse(message.content.toString('utf8'))); - this.channel.ack(message); - } - }) - } - - async send(responseTopic: string, msgKey: string, rawResponse: Buffer, headers: any): Promise { - - if (!this.topics.includes(responseTopic)) { - await this.createQueue(responseTopic); - this.topics.push(responseTopic); - } - - let data = JSON.stringify( - { - key: msgKey, - data: [...rawResponse], - headers: headers - }); - let dataBuffer = Buffer.from(data); - this.channel.sendToQueue(responseTopic, dataBuffer); - return this.channel.waitForConfirms() - } - - private parseQueueProperties() { - let args: { [n: string]: number } = {}; - const props = this.queueProperties.split(';'); - props.forEach(p => { - const delimiterPosition = p.indexOf(':'); - args[p.substring(0, delimiterPosition)] = Number(p.substring(delimiterPosition + 1)); - }); - this.queueOptions['arguments'] = args; - } - - private async createQueue(topic: string): Promise { - return this.channel.assertQueue(topic, this.queueOptions); - } - - async destroy() { - this.logger.info('Stopping RabbitMQ resources...'); - - if (this.channel) { - this.logger.info('Stopping RabbitMQ chanel...'); - const _channel = this.channel; - // @ts-ignore - delete this.channel; - await _channel.close(); - this.logger.info('RabbitMQ chanel stopped'); - } - - if (this.connection) { - this.logger.info('Stopping RabbitMQ connection...') - try { - const _connection = this.connection; - // @ts-ignore - delete this.connection; - await _connection.close(); - this.logger.info('RabbitMQ client connection.'); - } catch (e) { - this.logger.info('RabbitMQ connection stop error.'); - } - } - this.logger.info('RabbitMQ resources stopped.') - } - -} diff --git a/msa/js-executor/queue/serviceBusTemplate.ts b/msa/js-executor/queue/serviceBusTemplate.ts deleted file mode 100644 index 2a9c1d843e..0000000000 --- a/msa/js-executor/queue/serviceBusTemplate.ts +++ /dev/null @@ -1,175 +0,0 @@ -/// -/// Copyright © 2016-2024 The Thingsboard Authors -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// - -import config from 'config'; -import { _logger } from '../config/logger'; -import { JsInvokeMessageProcessor } from '../api/jsInvokeMessageProcessor' -import { IQueue } from './queue.models'; -import { - CreateQueueOptions, - ProcessErrorArgs, - ServiceBusAdministrationClient, - ServiceBusClient, - ServiceBusReceivedMessage, - ServiceBusReceiver, - ServiceBusSender -} from '@azure/service-bus'; - -export class ServiceBusTemplate implements IQueue { - - private logger = _logger(`serviceBusTemplate`); - private queuePrefix: string = config.get('queue_prefix'); - private requestTopic: string = this.queuePrefix ? this.queuePrefix + "." + config.get('request_topic') : config.get('request_topic'); - private namespaceName = config.get('service_bus.namespace_name'); - private sasKeyName = config.get('service_bus.sas_key_name'); - private sasKey = config.get('service_bus.sas_key'); - private queueProperties: string = config.get('service_bus.queue_properties'); - - private sbClient: ServiceBusClient; - private serviceBusService: ServiceBusAdministrationClient; - private queueOptions: CreateQueueOptions = {}; - private queues: string[] = []; - private receiver: ServiceBusReceiver; - private senderMap = new Map(); - - name = 'Azure Service Bus'; - - constructor() { - } - - async init() { - const connectionString = `Endpoint=sb://${this.namespaceName}.servicebus.windows.net/;SharedAccessKeyName=${this.sasKeyName};SharedAccessKey=${this.sasKey}`; - this.sbClient = new ServiceBusClient(connectionString) - this.serviceBusService = new ServiceBusAdministrationClient(connectionString); - - this.parseQueueProperties(); - - const listQueues = this.serviceBusService.listQueues(); - for await (const queue of listQueues) { - this.queues.push(queue.name); - } - - if (!this.queues.includes(this.requestTopic)) { - await this.createQueueIfNotExist(this.requestTopic); - this.queues.push(this.requestTopic); - } - - this.receiver = this.sbClient.createReceiver(this.requestTopic, {receiveMode: 'peekLock'}); - - const messageProcessor = new JsInvokeMessageProcessor(this); - - const messageHandler = async (message: ServiceBusReceivedMessage) => { - if (message) { - messageProcessor.onJsInvokeMessage(message.body); - await this.receiver.completeMessage(message); - } - }; - const errorHandler = async (error: ProcessErrorArgs) => { - this.logger.error('Failed to receive message from queue.', error); - }; - this.receiver.subscribe({processMessage: messageHandler, processError: errorHandler}) - } - - async send(responseTopic: string, msgKey: string, rawResponse: Buffer, headers: any): Promise { - if (!this.queues.includes(this.requestTopic)) { - await this.createQueueIfNotExist(this.requestTopic); - this.queues.push(this.requestTopic); - } - - let customSender = this.senderMap.get(responseTopic); - - if (!customSender) { - customSender = this.sbClient.createSender(responseTopic); - this.senderMap.set(responseTopic, customSender); - } - - let data = { - key: msgKey, - data: [...rawResponse], - headers: headers - }; - - return customSender.sendMessages({body: data}); - } - - private parseQueueProperties() { - let properties: { [n: string]: string } = {}; - const props = this.queueProperties.split(';'); - props.forEach(p => { - const delimiterPosition = p.indexOf(':'); - properties[p.substring(0, delimiterPosition)] = p.substring(delimiterPosition + 1); - }); - this.queueOptions = { - requiresDuplicateDetection: false, - maxSizeInMegabytes: Number(properties['maxSizeInMb']), - defaultMessageTimeToLive: `PT${properties['messageTimeToLiveInSec']}S`, - lockDuration: `PT${properties['lockDurationInSec']}S` - }; - } - - private async createQueueIfNotExist(topic: string) { - try { - await this.serviceBusService.createQueue(topic, this.queueOptions) - } catch (err: any) { - if (err && err.code !== "MessageEntityAlreadyExistsError") { - throw new Error(err); - } - } - } - - async destroy() { - this.logger.info('Stopping Azure Service Bus resources...') - if (this.receiver) { - this.logger.info('Stopping Service Bus Receiver...'); - try { - const _receiver = this.receiver; - // @ts-ignore - delete this.receiver; - await _receiver.close(); - this.logger.info('Service Bus Receiver stopped.'); - } catch (e) { - this.logger.info('Service Bus Receiver stop error.'); - } - } - - this.logger.info('Stopping Service Bus Senders...'); - const senders: Promise[] = []; - this.senderMap.forEach((sender) => { - senders.push(sender.close()); - }); - this.senderMap.clear(); - try { - await Promise.all(senders); - this.logger.info('Service Bus Senders stopped.'); - } catch (e) { - this.logger.info('Service Bus Senders stop error.'); - } - - if (this.sbClient) { - this.logger.info('Stopping Service Bus Client...'); - try { - const _sbClient = this.sbClient; - // @ts-ignore - delete this.sbClient; - await _sbClient.close(); - this.logger.info('Service Bus Client stopped.'); - } catch (e) { - this.logger.info('Service Bus Client stop error.'); - } - } - this.logger.info('Azure Service Bus resources stopped.') - } -} diff --git a/msa/js-executor/server.ts b/msa/js-executor/server.ts index 5bd1f59692..50d326d1e8 100644 --- a/msa/js-executor/server.ts +++ b/msa/js-executor/server.ts @@ -19,10 +19,6 @@ import { _logger } from './config/logger'; import { HttpServer } from './api/httpServer'; import { IQueue } from './queue/queue.models'; import { KafkaTemplate } from './queue/kafkaTemplate'; -import { PubSubTemplate } from './queue/pubSubTemplate'; -import { AwsSqsTemplate } from './queue/awsSqsTemplate'; -import { RabbitMqTemplate } from './queue/rabbitmqTemplate'; -import { ServiceBusTemplate } from './queue/serviceBusTemplate'; const logger = _logger('main'); @@ -55,14 +51,6 @@ async function createQueue(serviceType: string): Promise { switch (serviceType) { case 'kafka': return new KafkaTemplate(); - case 'pubsub': - return new PubSubTemplate(); - case 'aws-sqs': - return new AwsSqsTemplate(); - case 'rabbitmq': - return new RabbitMqTemplate(); - case 'service-bus': - return new ServiceBusTemplate(); default: throw new Error('Unknown service type: ' + serviceType); } diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 6b79f2d101..cba83e5ff7 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -47,9 +47,7 @@ zk: # Queue configuration parameters queue: - # kafka (Apache Kafka). The following queue types are deprecated and will no longer be supported in ThingsBoard 4.0: - # aws-sqs (AWS SQS), pubsub (PubSub), service-bus (Azure Service Bus), rabbitmq (RabbitMQ) - type: "${TB_QUEUE_TYPE:kafka}" + type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). in_memory: stats: @@ -148,91 +146,10 @@ queue: print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}" # Time to wait for the stats-loading requests to Kafka to finis kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}" - aws_sqs: - # Use the default credentials provider for AWS SQS - use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" - # Access key ID from AWS IAM user - access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" - # Secret access key from AWS IAM user - secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" - # Region from AWS account - region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" - # Number of threads per each AWS SQS queue in consumer - threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - # Thread pool size for aws_sqs queue producer executor provider. Default value equals to AmazonSQSAsyncClient.DEFAULT_THREAD_POOL_SIZE - producer_thread_pool_size: "${TB_QUEUE_AWS_SQS_EXECUTOR_THREAD_POOL_SIZE:50}" - queue-properties: - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - version-control: "${TB_QUEUE_AWS_SQS_VC_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - pubsub: - # Project ID from Google Cloud - project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" - # API Credentials in JSON format - service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - # Message size for PubSub queue.Value in bytes - max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes - # Number of messages per consumer - max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - # Thread pool size for pubsub queue executor provider. If not set - default pubsub executor provider value will be used (5 * number of available processors) - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:0}" - queue-properties: - # Pub/Sub properties for Core subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # Pub/Sub properties for Version Control subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # Pub/Sub properties for Transport Api subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - version-control: "${TB_QUEUE_PUBSUB_VC_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - service_bus: - # Azure namespace - namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" - # Azure Service Bus Shared Access Signatures key name - sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" - # Azure Service Bus Shared Access Signatures key - sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" - # Number of messages per a consumer - max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" - queue-properties: - # Azure Service Bus properties for Core queues - core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus properties for Notification queues - notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus properties for Version Control queues - version-control: "${TB_QUEUE_SERVICE_BUS_VC_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - rabbitmq: - # By default empty - exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" - # RabbitMQ host used to establish connection - host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" - # RabbitMQ host used to establish a connection - port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" - # Virtual hosts provide logical grouping and separation of resources - virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" - # Username for RabbitMQ user account - username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" - # User password for RabbitMQ user account - password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" - # Network connection between clients and RabbitMQ nodes can fail. RabbitMQ Java client supports automatic recovery of connections and topology (queues, exchanges, bindings, and consumers) - automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" - # The connection timeout for the RabbitMQ connection factory - connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" - # RabbitMQ has a timeout for connection handshake. When clients run in heavily constrained environments, it may be necessary to increase the timeout - handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" - queue-properties: - # RabbitMQ properties for Core queues - core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ properties for Transport API queues - transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ properties for Version Control queues - version-control: "${TB_QUEUE_RABBIT_MQ_VC_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 core: - # Default topic name of Kafka, RabbitMQ, etc. queue + # Default topic name topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" # Interval in milliseconds to poll messages by Core microservices poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" @@ -247,7 +164,7 @@ queue: pack-interval-ms: "${TB_QUEUE_CORE_OTA_PACK_INTERVAL_MS:60000}" # The size of OTA updates notifications fetched from the queue. The queue stores pairs of firmware and device ids pack-size: "${TB_QUEUE_CORE_OTA_PACK_SIZE:100}" - # Stats topic name for queue Kafka, RabbitMQ, etc. + # Stats topic name usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}" stats: # Enable/disable statistics for Core microservices @@ -258,7 +175,7 @@ queue: # Topic name for Housekeeper tasks topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" vc: - # Default topic name for Kafka, RabbitMQ, etc. + # Default topic name topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" # Number of partitions to associate with this queue. Used for scaling the number of messages that can be processed in parallel partitions: "${TB_QUEUE_VC_PARTITIONS:10}" @@ -266,7 +183,7 @@ queue: poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" # Timeout before retrying all failed and timed-out messages from the processing pack pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:180000}" - # Queue settings for Kafka, RabbitMQ, etc. Limit for single message size + # Limit for single queue message size msg-chunk-size: "${TB_QUEUE_VC_MSG_CHUNK_SIZE:250000}" # Version control parameters diff --git a/pom.xml b/pom.xml index dec169a9b6..e70a57fbbf 100755 --- a/pom.xml +++ b/pom.xml @@ -117,7 +117,6 @@ 1.12.701 1.128.1 2.37.1 - 3.6.7 1.6.4 1.6.1 1.9.4 @@ -2038,21 +2037,6 @@ proto-google-common-protos ${google.common.protos.version} - - com.microsoft.azure - azure-servicebus - ${azure-servicebus.version} - - - com.nimbusds - content-type - - - org.ow2.asm - asm - - - org.passay passay diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 0c7ff4e43e..66ab703a54 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -256,9 +256,7 @@ coap: # Queue configuration parameters queue: - # kafka (Apache Kafka). The following queue types are deprecated and will no longer be supported in ThingsBoard 4.0: - # aws-sqs (AWS SQS), pubsub (PubSub), service-bus (Azure Service Bus), rabbitmq (RabbitMQ) - type: "${TB_QUEUE_TYPE:kafka}" + type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). kafka: # Kafka Bootstrap Servers @@ -333,94 +331,6 @@ queue: notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Housekeeper tasks topic housekeeper: "${TB_QUEUE_KAFKA_HOUSEKEEPER_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" - aws_sqs: - # Use the default credentials provider for AWS SQS - use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" - # Access key ID from AWS IAM user - access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" - # Secret access key from AWS IAM user - secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" - # Region from AWS account - region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" - # Number of threads per each AWS SQS queue in consumer - threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - # Thread pool size for aws_sqs queue producer executor provider. Default value equals to AmazonSQSAsyncClient.DEFAULT_THREAD_POOL_SIZE - producer_thread_pool_size: "${TB_QUEUE_AWS_SQS_EXECUTOR_THREAD_POOL_SIZE:50}" - queue-properties: - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - pubsub: - # Project ID from Google Cloud - project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" - # API Credentials in JSON format - service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - # Message size for PubSub queue. Value in bytes - max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" - # Number of messages per consumer - max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - # Thread pool size for pubsub queue executor provider. If not set - default pubsub executor provider value will be used (5 * number of available processors) - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:0}" - queue-properties: - # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # Pub/Sub properties for Core subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # Pub/Sub properties for Transport API subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # Pub/Sub properties for Version Control subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - service_bus: - # Azure namespace - namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" - # Azure Service Bus Shared Access Signatures key name - sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" - # Azure Service Bus Shared Access Signatures key - sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" - # Number of messages per a consumer - max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" - queue-properties: - # Azure Service Bus properties for Rule Engine queues - rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus properties for Core queues - core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus properties for Transport Api queues - transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus properties for Notification queues - notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - rabbitmq: - # By default empty - exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" - # RabbitMQ host used to establish connection - host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" - # RabbitMQ host used to establish a connection - port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" - # Virtual hosts provide logical grouping and separation of resources - virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" - # Username for RabbitMQ user account - username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" - # User password for RabbitMQ user account - password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" - # Network connection between clients and RabbitMQ nodes can fail. RabbitMQ Java client supports automatic recovery of connections and topology (queues, exchanges, bindings, and consumers) - automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" - # The connection timeout for the RabbitMQ connection factory - connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" - # RabbitMQ has a timeout for connection handshake. When clients run in heavily constrained environments, it may be necessary to increase the timeout - handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" - queue-properties: - # RabbitMQ properties for Rule Engine queues - rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ properties for Core queues - core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ properties for Transport API queues - transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ properties for Notification queues - notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 transport_api: @@ -439,7 +349,7 @@ queue: # Interval in milliseconds to poll api response from transport microservices response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: - # Default topic name of Kafka, RabbitMQ, etc. queue + # Default topic name topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" # Interval in milliseconds to poll messages by Core microservices poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" @@ -447,7 +357,7 @@ queue: partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" # Timeout for processing a message pack by Core microservices pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" - # Default topic name for queue Kafka, RabbitMQ, etc. + # Default topic name usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}" stats: # Enable/disable statistics for Core microservices diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 3c28c6ddf6..be91e4fe67 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -202,9 +202,7 @@ transport: # Queue configuration parameters queue: - # kafka (Apache Kafka). The following queue types are deprecated and will no longer be supported in ThingsBoard 4.0: - # aws-sqs (AWS SQS), pubsub (PubSub), service-bus (Azure Service Bus), rabbitmq (RabbitMQ) - type: "${TB_QUEUE_TYPE:kafka}" + type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka) . kafka: # Kafka Bootstrap Servers @@ -278,95 +276,6 @@ queue: notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Housekeeper tasks topic housekeeper: "${TB_QUEUE_KAFKA_HOUSEKEEPER_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" - aws_sqs: - # Use default credentials provider for AWS SQS - use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" - # Access key ID from AWS IAM user - access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" - # Secret access key from AWS IAM user - secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" - # Region from AWS account - region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" - # Number of threads per each AWS SQS queue in consumer - threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - # Thread pool size for aws_sqs queue producer executor provider. Default value equals to AmazonSQSAsyncClient.DEFAULT_THREAD_POOL_SIZE - producer_thread_pool_size: "${TB_QUEUE_AWS_SQS_EXECUTOR_THREAD_POOL_SIZE:50}" - queue-properties: - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - pubsub: - # Project ID from Google Cloud - project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" - # API Credentials in JSON format - service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - # Message size for PubSub queue.Value in bytes - max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" - # Number of messages per a consumer - max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - # Thread pool size for pubsub queue executor provider. If not set - default pubsub executor provider value will be used (5 * number of available processors) - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:0}" - queue-properties: - # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consume again - rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # Pub/Sub properties for Core subscribers, messages which will commit after ackDeadlineInSec period can be consume again - core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # Pub/Sub properties for Transport API subscribers, messages which will commit after ackDeadlineInSec period can be consume again - transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # Pub/Sub properties for Version Control subscribers, messages which will commit after ackDeadlineInSec period can be consume again - notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - service_bus: - # Azure namespace - namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" - # Azure Service Bus Shared Access Signatures key name - sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" - # Azure Service Bus Shared Access Signatures key - sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" - # Number of messages per a consumer - max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" - queue-properties: - # Azure Service Bus properties for Rule Engine queues - rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus properties for Core queues - core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus properties for Transport Api queues - transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus properties for Notification queues - notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - rabbitmq: - # By default empty - exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" - # RabbitMQ host used to establish connection - host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" - # RabbitMQ host used to establish a connection - port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" - # Virtual hosts provide logical grouping and separation of resources - virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" - # Username for RabbitMQ user account - username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" - # User password for RabbitMQ user account - password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" - # Network connection between clients and RabbitMQ nodes can fail. RabbitMQ Java client supports automatic recovery of connections and topology (queues, exchanges, bindings, and consumers) - automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" - # The connection timeout for the RabbitMQ connection factory - connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" - # RabbitMQ has a timeout for connection handshake. When clients run in heavily constrained environments, it may be necessary to increase the timeout - handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" - queue-properties: - # RabbitMQ properties for Rule Engine queues - rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ properties for Core queues - core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ properties for Transport API queues - transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ properties for Notification queues - notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 transport_api: @@ -385,7 +294,7 @@ queue: # Interval in milliseconds to poll api response from transport microservices response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: - # Default topic name of Kafka, RabbitMQ, etc. queue + # Default topic name topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" # Interval in milliseconds to poll messages by Core microservices poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" @@ -393,7 +302,7 @@ queue: partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" # Timeout for processing a message pack by Core microservices pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" - # Default topic name for queue Kafka, RabbitMQ, etc. + # Default topic name usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}" stats: # Enable/disable statistics for Core microservices diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index bafe77ee92..7bdd68baf1 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -302,9 +302,7 @@ transport: # Queue configuration properties queue: - # kafka (Apache Kafka). The following queue types are deprecated and will no longer be supported in ThingsBoard 4.0: - # aws-sqs (AWS SQS), pubsub (PubSub), service-bus (Azure Service Bus), rabbitmq (RabbitMQ) - type: "${TB_QUEUE_TYPE:kafka}" + type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). kafka: # Kafka Bootstrap Servers @@ -379,94 +377,6 @@ queue: notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Housekeeper tasks topic housekeeper: "${TB_QUEUE_KAFKA_HOUSEKEEPER_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" - aws_sqs: - # Use the default credentials provider for AWS SQS - use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" - # Access key ID from AWS IAM user - access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" - # Secret access key from AWS IAM user - secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" - # Region from AWS account - region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" - # Number of threads per each AWS SQS queue in consumer - threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - # Thread pool size for aws_sqs queue producer executor provider. Default value equals to AmazonSQSAsyncClient.DEFAULT_THREAD_POOL_SIZE - producer_thread_pool_size: "${TB_QUEUE_AWS_SQS_EXECUTOR_THREAD_POOL_SIZE:50}" - queue-properties: - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - pubsub: - # Project ID from Google Cloud - project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" - # API Credentials in JSON format - service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - # Message size for PubSub queue. Value in bytes - max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" - # Number of messages per consumer - max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - # Thread pool size for pubsub queue executor provider. If not set - default pubsub executor provider value will be used (5 * number of available processors) - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:0}" - queue-properties: - # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # Pub/Sub properties for Core subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # Pub/Sub properties for Transport API subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # Pub/Sub properties for Version Control subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - service_bus: - # Azure namespace - namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" - # Azure Service Bus Shared Access Signatures key name - sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" - # Azure Service Bus Shared Access Signatures key - sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" - # Number of messages per a consumer - max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" - queue-properties: - # Azure Service Bus properties for Rule Engine queues - rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus properties for Core queues - core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus properties for Transport Api queues - transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus properties for Notification queues - notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - rabbitmq: - # By default empty - exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" - # RabbitMQ host used to establish connection - host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" - # RabbitMQ host used to establish a connection - port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" - # Virtual hosts provide logical grouping and separation of resources - virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" - # Username for RabbitMQ user account - username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" - # User password for RabbitMQ user account - password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" - # Network connection between clients and RabbitMQ nodes can fail. RabbitMQ Java client supports automatic recovery of connections and topology (queues, exchanges, bindings, and consumers) - automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" - # The connection timeout for the RabbitMQ connection factory - connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" - # RabbitMQ has a timeout for connection handshake. When clients run in heavily constrained environments, it may be necessary to increase the timeout - handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" - queue-properties: - # RabbitMQ properties for Rule Engine queues - rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ properties for Core queues - core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ properties for Transport API queues - transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ properties for Notification queues - notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 transport_api: @@ -485,7 +395,7 @@ queue: # Interval in milliseconds to poll api response from transport microservices response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: - # Default topic name of Kafka, RabbitMQ, etc. queue + # Default topic name topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" # Interval in milliseconds to poll messages by Core microservices poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 51a2c173a7..8d9a60a319 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -235,9 +235,7 @@ transport: # Queue configuration parameters queue: - # kafka (Apache Kafka). The following queue types are deprecated and will no longer be supported in ThingsBoard 4.0: - # aws-sqs (AWS SQS), pubsub (PubSub), service-bus (Azure Service Bus), rabbitmq (RabbitMQ) - type: "${TB_QUEUE_TYPE:kafka}" + type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). kafka: # Kafka Bootstrap Servers @@ -312,94 +310,6 @@ queue: notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Housekeeper tasks topic housekeeper: "${TB_QUEUE_KAFKA_HOUSEKEEPER_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" - aws_sqs: - # Use the default credentials provider for AWS SQS - use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" - # Access key ID from AWS IAM user - access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" - # Secret access key from AWS IAM user - secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" - # Region from AWS account - region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" - # Number of threads per each AWS SQS queue in consumer - threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - # Thread pool size for aws_sqs queue producer executor provider. Default value equals to AmazonSQSAsyncClient.DEFAULT_THREAD_POOL_SIZE - producer_thread_pool_size: "${TB_QUEUE_AWS_SQS_EXECUTOR_THREAD_POOL_SIZE:50}" - queue-properties: - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - pubsub: - # Project ID from Google Cloud - project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" - # API Credentials in JSON format - service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - # Message size for PubSub queue.Value in bytes - max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" - # Number of messages per consumer - max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - # Thread pool size for pubsub queue executor provider. If not set - default pubsub executor provider value will be used (5 * number of available processors) - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:0}" - queue-properties: - # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # Pub/Sub properties for Core subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # Pub/Sub properties for Transport API subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # Pub/Sub properties for Version Control subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - service_bus: - # Azure namespace - namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" - # Azure Service Bus Shared Access Signatures key name - sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" - # Azure Service Bus Shared Access Signatures key - sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" - # Number of messages per a consumer - max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" - queue-properties: - # Azure Service Bus properties for Rule Engine queues - rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus properties for Core queues - core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus properties for Transport Api queues - transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus properties for Notification queues - notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - rabbitmq: - # By default empty - exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" - # RabbitMQ host used to establish connection - host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" - # RabbitMQ host used to establish a connection - port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" - # Virtual hosts provide logical grouping and separation of resources - virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" - # Username for RabbitMQ user account - username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" - # User password for RabbitMQ user account - password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" - # Network connection between clients and RabbitMQ nodes can fail. RabbitMQ Java client supports automatic recovery of connections and topology (queues, exchanges, bindings, and consumers) - automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" - # The connection timeout for the RabbitMQ connection factory - connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" - # RabbitMQ has a timeout for connection handshake. When clients run in heavily constrained environments, it may be necessary to increase the timeout - handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" - queue-properties: - # RabbitMQ properties for Rule Engine queues - rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ properties for Core queues - core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ properties for Transport API queues - transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ properties for Notification queues - notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 transport_api: @@ -418,7 +328,7 @@ queue: # Interval in milliseconds to poll api response from transport microservices response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: - # Default topic name of Kafka, RabbitMQ, etc. queue + # Default topic name topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" # Interval in milliseconds to poll messages by Core microservices poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" @@ -426,7 +336,7 @@ queue: partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" # Timeout for processing a message pack by Core microservices pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" - # Default topic name for queue Kafka, RabbitMQ, etc. + # Default topic name usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}" stats: # Enable/disable statistics for Core microservices diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index df222fe09f..4c11bd0018 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -181,9 +181,7 @@ transport: # Queue configuration parameters queue: - # kafka (Apache Kafka). The following queue types are deprecated and will no longer be supported in ThingsBoard 4.0: - # aws-sqs (AWS SQS), pubsub (PubSub), service-bus (Azure Service Bus), rabbitmq (RabbitMQ) - type: "${TB_QUEUE_TYPE:kafka}" + type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). kafka: # Kafka Bootstrap Servers @@ -265,94 +263,6 @@ queue: print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}" # Time to wait for the stats-loading requests to Kafka to finis kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}" - aws_sqs: - # Use the default credentials provider for AWS SQS - use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" - # Access key ID from AWS IAM user - access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" - # Secret access key from AWS IAM user - secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" - # Region from AWS account - region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" - # Number of threads per each AWS SQS queue in consumer - threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" - # Thread pool size for aws_sqs queue producer executor provider. Default value equals to AmazonSQSAsyncClient.DEFAULT_THREAD_POOL_SIZE - producer_thread_pool_size: "${TB_QUEUE_AWS_SQS_EXECUTOR_THREAD_POOL_SIZE:50}" - queue-properties: - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds - notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - pubsub: - # Project ID from Google Cloud - project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" - # API Credentials in JSON format - service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - # Message size for PubSub queue. Value in bytes - max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" - # Number of messages per consumer - max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" - # Thread pool size for pubsub queue executor provider. If not set - default pubsub executor provider value will be used (5 * number of available processors) - executor_thread_pool_size: "${TB_QUEUE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:0}" - queue-properties: - # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # Pub/Sub properties for Core subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # Pub/Sub properties for Transport API subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - # Pub/Sub properties for Version Control subscribers, messages which will commit after ackDeadlineInSec period can be consumed again - notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - service_bus: - # Azure namespace - namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" - # Azure Service Bus Shared Access Signatures key name - sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" - # Azure Service Bus Shared Access Signatures key - sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" - # Number of messages per a consumer - max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" - queue-properties: - # Azure Service Bus properties for Rule Engine queues - rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus properties for Core queues - core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus properties for Transport Api queues - transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - # Azure Service Bus properties for Notification queues - notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - rabbitmq: - # By default empty - exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" - # RabbitMQ host used to establish connection - host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" - # RabbitMQ host used to establish a connection - port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" - # Virtual hosts provide logical grouping and separation of resources - virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" - # Username for RabbitMQ user account - username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" - # User password for RabbitMQ user account - password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" - # Network connection between clients and RabbitMQ nodes can fail. RabbitMQ Java client supports automatic recovery of connections and topology (queues, exchanges, bindings, and consumers) - automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" - # The connection timeout for the RabbitMQ connection factory - connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" - # RabbitMQ has a timeout for connection handshake. When clients run in heavily constrained environments, it may be necessary to increase the timeout - handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" - queue-properties: - # RabbitMQ properties for Rule Engine queues - rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ properties for Core queues - core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ properties for Transport API queues - transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - # RabbitMQ properties for Notification queues - notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 transport_api: @@ -371,7 +281,7 @@ queue: # Interval in milliseconds to poll api response from transport microservices response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: - # Default topic name of Kafka, RabbitMQ, etc. queue + # Default topic name topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" # Interval in milliseconds to poll messages by Core microservices poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" @@ -379,7 +289,7 @@ queue: partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" # Timeout for processing a message pack by Core microservices pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" - # Stats topic name for queue Kafka, RabbitMQ, etc. + # Stats topic name usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}" stats: # Enable/disable statistics for Core microservices